Compare commits

..
2 Commits
Author SHA1 Message Date
MediaPipe Teamandjqtang 3b6d3c4058 Project import generated by Copybara.
GitOrigin-RevId: 4419aaa472eeb91123d1f8576188166ee0e5ea69
2020-03-10 18:14:25 -07:00
MediaPipe Teamandjqtang 252a5713c7 Project import generated by Copybara.
GitOrigin-RevId: 6f964e58d874e47fb6207aa97d060a4cd6428527
2020-03-02 10:35:07 -08:00
229 changed files with 24106 additions and 962 deletions
+4 -1
View File
@@ -15,6 +15,7 @@
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
* [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
* [AutoFlip](mediapipe/docs/autoflip.md)
![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
@@ -43,6 +44,8 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
## Publications
* [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
* [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)
* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
@@ -63,7 +66,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
## Alpha Disclaimer
MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
MediaPipe is currently in alpha for v0.7. We are still making breaking API changes and expect to get to stable API by v1.0.
## Contributing
We welcome contributions. Please follow these [guidelines](./CONTRIBUTING.md).
+11 -10
View File
@@ -10,15 +10,15 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1",
versions.check(minimum_bazel_version = "1.0.0",
maximum_bazel_version = "1.2.1")
# ABSL cpp library lts_2019_08_08.
# ABSL cpp library lts_2020_02_25
http_archive(
name = "com_google_absl",
urls = [
"https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
],
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [
@@ -27,8 +27,8 @@ http_archive(
patch_args = [
"-p1",
],
strip_prefix = "abseil-cpp-20190808",
sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
strip_prefix = "abseil-cpp-20200225",
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
)
http_archive(
@@ -117,18 +117,19 @@ http_archive(
],
)
# 2019-11-21
_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
# 2020-02-12
# The last commit before TensorFlow switched to Bazel 2.0
_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
http_archive(
name = "org_tensorflow",
urls = [
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
],
# Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
# A compatibility patch
patches = [
"@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
"@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
],
patch_args = [
"-p1",
+16
View File
@@ -610,6 +610,22 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "side_packet_to_stream_calculator_test",
srcs = ["side_packet_to_stream_calculator_test.cc"],
deps = [
":side_packet_to_stream_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "immediate_mux_calculator_test",
srcs = ["immediate_mux_calculator_test.cc"],
@@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet;
using mediapipe::Timestamp;
namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK";
static std::map<std::string, Timestamp>* kTimestampMap = []() {
auto* res = new std::map<std::string, Timestamp>();
res->emplace("AT_PRESTREAM", Timestamp::PreStream());
res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
res->emplace("AT_ZERO", Timestamp(0));
res->emplace(kTagAtPreStream, Timestamp::PreStream());
res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset());
return res;
}();
template <typename CC>
std::string GetOutputTag(const CC& cc) {
// Single output tag only is required by contract.
return *cc.Outputs().GetTags().begin();
}
} // namespace
// Outputs the single input_side_packet at the timestamp specified in the
// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
// Outputs side packet(s) in corresponding output stream(s) with a particular
// timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.)
//
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and
// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(),
// Timestamp(0) and timestamp of a packet received in TICK input.
//
// Examples:
// node {
// calculator: "SidePacketToStreamCalculator"
// input_side_packet: "side_packet"
// output_stream: "AT_PRESTREAM:packet"
// }
//
// node {
// calculator: "SidePacketToStreamCalculator"
// input_stream: "TICK:tick"
// input_side_packet: "side_packet"
// output_stream: "AT_TICK:packet"
// }
class SidePacketToStreamCalculator : public CalculatorBase {
public:
SidePacketToStreamCalculator() = default;
~SidePacketToStreamCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
bool is_tick_processing_ = false;
std::string output_tag_;
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
CalculatorContract* cc) {
cc->InputSidePackets().Index(0).SetAny();
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is "
"allowed and required to specify output stream(s).";
RET_CHECK(
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
<< "Either both of TICK and AT_TICK should be used or none of them.";
const std::string output_tag = GetOutputTag(*cc);
const int num_entries = cc->Outputs().NumEntries(output_tag);
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
<< "Same number of input side packets and output streams is required.";
for (int i = 0; i < num_entries; ++i) {
cc->InputSidePackets().Index(i).SetAny();
cc->Outputs()
.Get(output_tag, i)
.SetSameAs(cc->InputSidePackets().Index(i).GetSameAs());
}
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
if (cc->Inputs().HasTag(kTagTick)) {
cc->Inputs().Tag(kTagTick).SetAny();
}
RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
cc->Outputs().Tag(*tags.begin()).SetAny();
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
output_tag_ = GetOutputTag(*cc);
if (cc->Inputs().HasTag(kTagTick)) {
is_tick_processing_ = true;
// Set offset, so output timestamp bounds are updated in response to TICK
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Process(
CalculatorContext* cc) {
return mediapipe::tool::StatusStop();
if (is_tick_processing_) {
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
return ::mediapipe::OkStatus();
}
return ::mediapipe::tool::StatusStop();
}
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
const std::string& tag = *tags.begin();
RET_CHECK_EQ(kTimestampMap->count(tag), 1);
cc->Outputs().Tag(tag).AddPacket(
cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
if (!cc->Outputs().HasTag(kTagAtTick)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
}
return ::mediapipe::OkStatus();
}
@@ -0,0 +1,275 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
namespace mediapipe {
namespace {
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Either both of TICK and AT_TICK should be used or none of them.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "DOES_NOT_EXIST:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:packet0"
output_stream: "AT_PRE_STREAM:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
output_stream: "AT_PRESTREAM:0:packet0"
output_stream: "AT_PRESTREAM:1:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_PRESTREAM:packet0"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
void DoTestNonAtTickOutputTag(absl::string_view tag,
Timestamp expected_timestamp) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::StrReplaceAll(
R"(
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "$tag:packet"
}
)",
{{"$tag", tag}}));
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 10;
std::vector<Packet> output_packets;
MP_ASSERT_OK(graph.ObserveOutputStream(
"packet", [&output_packets](const Packet& packet) {
output_packets.push_back(packet);
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
MP_ASSERT_OK(graph.WaitForObservedOutput());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
}
TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream());
DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream());
DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0));
}
TEST(SidePacketToStreamCalculator, AtTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto tick_and_verify = [&graph, &output_packets,
expected_value](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:0:packet0"
output_stream: "AT_TICK:1:packet1"
}
)");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto tick_and_verify = [&graph, &output_packets0, &output_packets1,
expected_value0, expected_value1](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets0.empty());
ASSERT_FALSE(output_packets1.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp());
EXPECT_EQ(expected_value0, output_packets0.back().Get<int>());
EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp());
EXPECT_EQ(expected_value1, output_packets1.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
} // namespace
} // namespace mediapipe
+1 -3
View File
@@ -346,9 +346,7 @@ cc_library(
],
"//conditions:default": [],
}),
visibility = [
"//visibility:public",
],
visibility = ["//visibility:public"],
deps = [
":image_cropping_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
@@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase {
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
private:
// Wrangles the appropriate inputs and outputs to perform the color
// conversion. The ImageFrame on input_tag is converted using the
@@ -75,11 +75,28 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
}
#endif // !MEDIAPIPE_DISABLE_GPU
RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag) ^
(cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
.has_norm_width() &&
cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
.has_norm_height()));
int flags = 0;
if (cc->Inputs().HasTag(kRectTag)) {
++flags;
}
if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
++flags;
}
if (cc->Inputs().HasTag(kNormRectTag)) {
++flags;
}
if (cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
.has_norm_width() &&
cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
.has_norm_height()) {
++flags;
}
if (cc->Options<mediapipe::ImageCroppingCalculatorOptions>().has_width() &&
cc->Options<mediapipe::ImageCroppingCalculatorOptions>().has_height()) {
++flags;
}
RET_CHECK(flags == 1) << "Illegal combination of input streams/options.";
if (cc->Inputs().HasTag(kRectTag)) {
cc->Inputs().Tag(kRectTag).Set<Rect>();
}
@@ -119,6 +136,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
// Validate border mode.
if (use_gpu_) {
MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc));
} else {
MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
}
return ::mediapipe::OkStatus();
}
@@ -162,6 +186,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
CalculatorContext* cc) {
int border_mode;
return GetBorderModeForOpenCV(cc, &border_mode);
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
CalculatorContext* cc) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
<< "implementation and will fall back into BORDER_REPLICATE";
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for GPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
return ::mediapipe::OkStatus();
@@ -172,6 +222,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
GetCropSpecs(cc, input_img.Width(), input_img.Height());
// Get border mode and value for OpenCV.
int border_mode;
MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode));
const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
cv::Size2f(target_width, target_height),
rotation * 180.f / M_PI);
@@ -191,7 +245,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
cv::getPerspectiveTransform(src_points, dst_points);
cv::Mat cropped_image;
cv::warpPerspective(input_mat, cropped_image, projection_matrix,
cv::Size(min_rect.size.width, min_rect.size.height));
cv::Size(min_rect.size.width, min_rect.size.height),
/* flags = */ 0,
/* borderMode = */ border_mode);
std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
input_img.Format(), cropped_image.cols, cropped_image.rows));
@@ -453,6 +509,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
rotation = options.rotation();
}
}
return {
.width = crop_width,
.height = crop_height,
@@ -462,4 +519,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
};
}
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
CalculatorContext* cc, int* border_mode) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
*border_mode = cv::BORDER_CONSTANT;
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
*border_mode = cv::BORDER_REPLICATE;
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for CPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
@@ -36,6 +36,7 @@
// Note: input_stream values take precedence over options defined in the graph.
//
namespace mediapipe {
struct RectSpec {
int width;
int height;
@@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase {
int src_height);
private:
::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
void GlRender();
void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
int* dst_width, int* dst_height);
::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
int* border_mode);
mediapipe::ImageCroppingCalculatorOptions options_;
@@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions {
// The (0, 0) point is at the (top, left) corner.
optional float norm_center_x = 6 [default = 0];
optional float norm_center_y = 7 [default = 0];
enum BorderMode {
// First unspecified value is required by the guideline. See details here:
// https://developers.google.com/protocol-buffers/docs/style#enums
BORDER_UNSPECIFIED = 0;
BORDER_ZERO = 1;
BORDER_REPLICATE = 2;
}
// Specifies behaviour for crops that go beyond image borders.
optional BorderMode border_mode = 8 [default = BORDER_ZERO];
}
@@ -56,11 +56,11 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
.width = 60,
@@ -69,9 +69,9 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when (width height) + (norm_width norm_height) are set in options.
@@ -96,11 +96,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
.width = 50,
.height = 50,
@@ -108,9 +108,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when WIDTH HEIGHT are set from input stream,
@@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"HEIGHT:0:crop_height",
"WIDTH:0:crop_width",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
RectSpec expectRect = {
@@ -157,9 +157,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
// Test when RECT is set from input stream,
@@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"RECT:0:rect",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
R"(
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
@@ -207,9 +207,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
.center_y = 40,
.rotation = 0.5,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
expectRect);
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
} // namespace
@@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// to be a multiple of 90 degrees. If provided, it overrides the
// ROTATION_DEGREES input side packet.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If
// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or
// corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If
// provided, it overrides the FLIP_VERTICALLY input side packet and/or
// corresponding field in the calculator options.
//
// Output:
// One of the following two tags:
// IMAGE - ImageFrame representing the output image.
@@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// degrees. It has to be a multiple of 90 degrees. It overrides the
// corresponding field in the calculator options.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not.
// It overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not.
// It overrides the corresponding field in the calculator options.
//
// Calculator options (see image_transformation_calculator.proto):
// output_width, output_height - (optional) Desired scaled image size.
// rotation_mode - (optional) Rotation in multiples of 90 degrees.
@@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase {
int output_height_ = 0;
mediapipe::RotationMode_Mode rotation_;
mediapipe::ScaleMode_Mode scale_mode_;
bool flip_horizontally_ = false;
bool flip_vertically_ = false;
bool use_gpu_ = false;
#if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
@@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
cc->InputSidePackets().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
@@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
output_width_ = options_.output_width();
output_height_ = options_.output_height();
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
@@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
rotation_ = options_.rotation_mode();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
flip_horizontally_ =
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>();
} else {
flip_horizontally_ = options_.flip_horizontally();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
flip_vertically_ =
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>();
} else {
flip_vertically_ = options_.flip_vertically();
}
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
if (use_gpu_) {
@@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
::mediapipe::Status ImageTransformationCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
!cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
rotation_ =
DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get<int>());
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return helper_.RunInGlContext(
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return RenderCpu(cc);
}
return ::mediapipe::OkStatus();
@@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
.Add(padding.release(), cc->InputTimestamp());
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
cv::Mat rotated_mat;
const int angle = RotationModeToDegrees(rotation_);
cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0);
@@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
cv::Mat flipped_mat;
if (options_.flip_horizontally() || options_.flip_vertically()) {
if (flip_horizontally_ || flip_vertically_) {
const int flip_code =
options_.flip_horizontally() && options_.flip_vertically()
? -1
: options_.flip_horizontally();
flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
cv::flip(rotated_mat, flipped_mat, flip_code);
} else {
flipped_mat = rotated_mat;
@@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
}
RET_CHECK(renderer) << "Unsupported input texture type";
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
scale_mode_, mediapipe::FrameScaleMode::kStretch);
mediapipe::FrameRotation rotation =
@@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, options_.flip_horizontally(), options_.flip_vertically(),
rotation, flip_horizontally_, flip_vertically_,
/*flip_texture=*/false));
glActiveTexture(GL_TEXTURE1);
@@ -260,11 +260,11 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
&crop_width_, &crop_height_, //
&col_start_, &row_start_));
MP_RETURN_IF_ERROR(
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
options_.target_width(), //
options_.target_height(), //
options_.preserve_aspect_ratio(), //
options_.scale_to_multiple_of_two(), //
scale_image::FindOutputDimensions(crop_width_, crop_height_, //
options_.target_width(), //
options_.target_height(), //
options_.preserve_aspect_ratio(), //
options_.scale_to_multiple_of(), //
&output_width_, &output_height_));
MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
&interpolation_algorithm_));
@@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
output_format_ = input_format_;
}
const bool is_positive_and_even =
(options_.scale_to_multiple_of() >= 1) &&
(options_.scale_to_multiple_of() % 2 == 0);
if (output_format_ == ImageFormat::YCBCR420P) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(is_positive_and_even)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when output format is YCbCr420P. To scale to "
"width and height of odd numbers, the output format must be SRGB.";
} else if (options_.preserve_aspect_ratio()) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(options_.scale_to_multiple_of() == 2)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when perserving aspect ratio. To scale to width "
"and height of odd numbers, please set "
"preserve_aspect_ratio to false.";
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (input_width_ > 0 && input_height_ > 0 &&
@@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto";
// 2) Scale and convert the image to fit inside target_width x target_height
// using the specified scaling algorithm. (maintaining the aspect
// ratio if preserve_aspect_ratio is true).
// The output width and height will be divisible by 2. It is possible to output
// width and height that are odd number when the output format is SRGB and not
// perserving the aspect ratio. See scale_to_multiple_of_two option for details.
// The output width and height will be divisible by 2, by default. It is
// possible to output width and height that are odd numbers when the output
// format is SRGB and the aspect ratio is left unpreserved. See
// scale_to_multiple_of for details.
message ScaleImageCalculatorOptions {
extend CalculatorOptions {
optional ScaleImageCalculatorOptions ext = 66237115;
@@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions {
// depending on the other options below. If unset, use the same width
// or height as the input. If only one is set then determine the other
// from the aspect ratio (after cropping). The output width and height
// will be divisible by 2.
// will be divisible by 2, by default.
optional int32 target_width = 1;
optional int32 target_height = 2;
@@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions {
// fits inside the box represented by target_width and target_height.
// Otherwise it is scaled to fit target_width and target_height
// completely. In any case, the aspect ratio that is preserved is
// that after cropping to the minimum/maximum aspect ratio.
// that after cropping to the minimum/maximum aspect ratio. Additionally, if
// true, the output width and height will be divisible by 2.
optional bool preserve_aspect_ratio = 3 [default = true];
// If ratio is positive, crop the image to this minimum and maximum
@@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions {
// SRGB or YCBCR420P.
optional ImageFormat.Format input_format = 12;
// If true, the output width and height will be divisible by 2. Otherwise it
// will use the exact specified output width and height, which is only
// supported when the output format is SRGB and preserve_aspect_ratio option
// is set to false.
optional bool scale_to_multiple_of_two = 13 [default = true];
// If set to 2, the target width and height will be rounded-down
// to the nearest even number. If set to any positive value other than 2,
// preserve_aspect_ratio must be false and the target width and height will be
// rounded-down to multiples of the given value. If set to any value less than
// 1, it will be treated like 1.
// NOTE: If set to an odd number, the output format must be SRGB.
optional int32 scale_to_multiple_of = 13 [default = 2];
// If true, assume the input YUV is BT.709 (this is the HDTV standard, so most
// content is likely using it). If false use the previous assumption of BT.601
@@ -88,17 +88,27 @@ double ParseRational(const std::string& rational) {
return ::mediapipe::OkStatus();
}
::mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
::mediapipe::Status FindOutputDimensions(int input_width, //
int input_height, //
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width,
int* output_height) {
CHECK(output_width);
CHECK(output_height);
if (preserve_aspect_ratio) {
RET_CHECK(scale_to_multiple_of == 2)
<< "FindOutputDimensions always outputs width and height that are "
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (scale_to_multiple_of < 1) scale_to_multiple_of = 1;
if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) {
if (target_width <= 0) {
target_width = input_width;
@@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) {
if (target_height <= 0) {
target_height = input_height;
}
if (scale_to_multiple_of_two) {
*output_width = (target_width / 2) * 2;
*output_height = (target_height / 2) * 2;
} else {
*output_width = target_width;
*output_height = target_height;
}
target_width -= target_width % scale_to_multiple_of;
target_height -= target_height % scale_to_multiple_of;
*output_width = target_width;
*output_height = target_height;
return ::mediapipe::OkStatus();
}
@@ -35,17 +35,19 @@ namespace scale_image {
int* col_start, int* row_start);
// Given an input width and height, a target width and height, whether to
// preserve the aspect ratio, and whether to round down to a multiple of 2,
// determine the output width and height. If target_width or target_height is
// non-positive, then they will be set to the input_width and input_height
// respectively. The output_width and output_height will be reduced as necessary
// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are
// specified.
// preserve the aspect ratio, and whether to round-down to the multiple of a
// given number nearest to the targets, determine the output width and height.
// If target_width or target_height is non-positive, then they will be set to
// the input_width and input_height respectively. If scale_to_multiple_of is
// less than 1, it will be treated like 1. The output_width and
// output_height will be reduced as necessary to preserve_aspect_ratio if the
// option is specified. If preserving the aspect ratio is desired, you must set
// scale_to_multiple_of to 2.
::mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
int target_width,
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
int target_height, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height);
} // namespace scale_image
@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
int output_width;
int output_height;
// Not scale.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(100, output_height);
// Not scale with odd input size.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
&output_height));
EXPECT_EQ(201, output_width);
EXPECT_EQ(101, output_height);
// Scale down by 1/2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale up, doubling dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
// visible by 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(74, output_height);
// Fits a 2:1 image into a 400 x 50 box.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to odd size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
&output_height));
EXPECT_EQ(151, output_width);
EXPECT_EQ(101, output_height);
}
@@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
int output_width;
int output_height;
// Scale width only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(100, output_height);
// Scale height only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Scale both dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(200, output_height);
}
// Tests scale_to_multiple_of.
TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
int output_width;
int output_height;
// Set no targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set width target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set height target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Set both targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
&output_height));
EXPECT_EQ(144, output_width);
EXPECT_EQ(200, output_height);
// Doesn't throw error if keep aspect is true and downscale multiple is 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Throws error if keep aspect is true, but downscale multiple is not 2.
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
&output_height),
testing::Not(testing::status::IsOk()));
// Downscaling to multiple ignored if multiple is less than 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
&output_height));
EXPECT_EQ(401, output_width);
EXPECT_EQ(201, output_height);
}
} // namespace
} // namespace scale_image
} // namespace mediapipe
+31 -21
View File
@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
srcs = ["image_frame_to_tensor_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":image_frame_to_tensor_calculator_proto"],
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
srcs = ["pack_media_sequence_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":pack_media_sequence_calculator_proto"],
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
cc_deps = [
"//mediapipe/framework:packet_generator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
srcs = ["vector_int_to_tensor_calculator_options.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":vector_int_to_tensor_calculator_options_proto"],
@@ -408,7 +408,7 @@ cc_library(
"//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -423,7 +423,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -654,7 +654,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:lib",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -695,7 +695,7 @@ cc_library(
"//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -737,7 +737,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@@ -745,6 +745,7 @@ cc_library(
cc_test(
name = "graph_tensors_packet_generator_test",
srcs = ["graph_tensors_packet_generator_test.cc"],
linkstatic = 1,
deps = [
":graph_tensors_packet_generator",
":graph_tensors_packet_generator_cc_proto",
@@ -761,6 +762,7 @@ cc_test(
name = "image_frame_to_tensor_calculator_test",
size = "small",
srcs = ["image_frame_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":image_frame_to_tensor_calculator",
"//mediapipe/framework:calculator_framework",
@@ -777,6 +779,7 @@ cc_test(
name = "matrix_to_tensor_calculator_test",
size = "small",
srcs = ["matrix_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":matrix_to_tensor_calculator",
":matrix_to_tensor_calculator_options_cc_proto",
@@ -793,6 +796,7 @@ cc_test(
name = "lapped_tensor_buffer_calculator_test",
size = "small",
srcs = ["lapped_tensor_buffer_calculator_test.cc"],
linkstatic = 1,
deps = [
":lapped_tensor_buffer_calculator",
":lapped_tensor_buffer_calculator_cc_proto",
@@ -801,7 +805,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -840,7 +844,7 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -867,7 +871,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -897,7 +901,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@@ -956,6 +960,7 @@ cc_test(
cc_test(
name = "tensor_squeeze_dimensions_calculator_test",
srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_squeeze_dimensions_calculator",
":tensor_squeeze_dimensions_calculator_cc_proto",
@@ -963,7 +968,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -971,6 +976,7 @@ cc_test(
name = "tensor_to_image_frame_calculator_test",
size = "small",
srcs = ["tensor_to_image_frame_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_image_frame_calculator",
":tensor_to_image_frame_calculator_cc_proto",
@@ -979,7 +985,7 @@ cc_test(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -987,6 +993,7 @@ cc_test(
name = "tensor_to_matrix_calculator_test",
size = "small",
srcs = ["tensor_to_matrix_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_matrix_calculator",
":tensor_to_matrix_calculator_cc_proto",
@@ -996,13 +1003,14 @@ cc_test(
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "tensor_to_vector_float_calculator_test",
srcs = ["tensor_to_vector_float_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_vector_float_calculator",
":tensor_to_vector_float_calculator_options_cc_proto",
@@ -1010,7 +1018,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -1030,13 +1038,14 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_int_to_tensor_calculator_test",
srcs = ["vector_int_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_int_to_tensor_calculator",
":vector_int_to_tensor_calculator_options_cc_proto",
@@ -1044,13 +1053,14 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_float_to_tensor_calculator_test",
srcs = ["vector_float_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_float_to_tensor_calculator",
":vector_float_to_tensor_calculator_options_cc_proto",
@@ -1058,7 +1068,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
@@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/deps/file_path.h"
@@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
+1
View File
@@ -485,6 +485,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],
@@ -148,7 +148,7 @@ struct GPUData {
// options: {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "modelname.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@@ -163,6 +163,9 @@ struct GPUData {
//
class TfLiteInferenceCalculator : public CalculatorBase {
public:
using TfLiteDelegatePtr =
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
@@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
std::unique_ptr<tflite::Interpreter> interpreter_;
std::unique_ptr<tflite::FlatBufferModel> model_;
TfLiteDelegate* delegate_ = nullptr;
TfLiteDelegatePtr delegate_;
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
@@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
cc->Outputs().HasTag("TENSORS_GPU"));
bool use_gpu = false;
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
bool use_gpu =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
if (cc->Inputs().HasTag("TENSORS"))
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Inputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU input is compatible with GPU delegate only.";
cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Outputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU output is compatible with GPU delegate only.";
cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
.Set<tflite::ops::builtin::BuiltinOpResolver>();
}
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
use_gpu |= options.use_gpu();
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
@@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
TfLiteGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
return ::mediapipe::OkStatus();
}));
#elif defined(MEDIAPIPE_IOS)
TFLGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@@ -472,8 +480,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i].reset();
}
#endif
} else {
delegate_ = nullptr;
}
delegate_ = nullptr;
}
#if defined(MEDIAPIPE_EDGE_TPU)
edgetpu_context_.reset();
@@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
}
// Get execution modes.
gpu_inference_ = options.use_gpu();
gpu_inference_ =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
return ::mediapipe::OkStatus();
}
@@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(interpreter_);
#if defined(__EMSCRIPTEN__)
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
interpreter_->SetNumThreads(1);
#else
interpreter_->SetNumThreads(
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.cpu_num_thread());
#endif // __EMSCRIPTEN__
if (gpu_output_) {
@@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
CalculatorContext* cc) {
#if defined(MEDIAPIPE_ANDROID)
const auto& calculator_opts =
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
if (calculator_opts.has_delegate() &&
calculator_opts.delegate().has_tflite()) {
// Default tflite inference requeqsted - no need to modify graph.
return ::mediapipe::OkStatus();
}
if (!gpu_inference_) {
if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.use_nnapi()) {
#if defined(MEDIAPIPE_ANDROID)
const bool nnapi_requested = calculator_opts.has_delegate()
? calculator_opts.delegate().has_nnapi()
: calculator_opts.use_nnapi();
if (nnapi_requested) {
// Attempt to use NNAPI.
// If not supported, the default CPU delegate will be created and used.
interpreter_->SetAllowFp16PrecisionForFp32(1);
delegate_ = tflite::NnApiDelegate();
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
delegate_ =
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
// No need to free according to tflite::NnApiDelegate()
// documentation.
});
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
return ::mediapipe::OkStatus();
}
#endif // MEDIAPIPE_ANDROID
// Return, no need for GPU delegate below.
return ::mediapipe::OkStatus();
}
#endif // ANDROID
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Configure and create the delegate.
@@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLITE_GL_OBJECT_TYPE_FASTEST;
options.compile_options.dynamic_batch_enabled = 0;
options.compile_options.inline_parameters = 1;
if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
&TfLiteGpuDelegateDelete);
if (gpu_input_) {
// Get input image sizes.
@@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_in_[i]->buffer.id(),
delegate_.get(), gpu_data_in_[i]->buffer.id(),
interpreter_->inputs()[i]),
kTfLiteOk);
}
@@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
for (int i = 0; i < gpu_data_out_.size(); ++i) {
RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
RET_CHECK_EQ(
TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
kTfLiteOk);
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_.get(), gpu_data_out_[i]->buffer.id(),
output_indices[i]),
kTfLiteOk);
}
}
// Must call this last.
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
#endif // OpenGL
#if defined(MEDIAPIPE_IOS)
@@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLGpuDelegateOptions options;
options.allow_precision_loss = true;
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
&TFLGpuDelegateDelete);
id<MTLDevice> device = gpu_helper_.mtlDevice;
if (gpu_input_) {
@@ -678,10 +714,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_in_[i]->buffer =
[device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, input_indices[i], gpu_data_in_[i]->buffer),
true);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer),
true);
}
}
if (gpu_output_) {
@@ -725,9 +763,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i]->buffer =
[device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, output_indices[i], gpu_data_out_[i]->buffer),
true);
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer),
true);
}
// Create converter for GPU output.
@@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto";
// options {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "model.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions {
optional TfLiteInferenceCalculatorOptions ext = 233867213;
}
message Delegate {
// Default inference provided by tflite.
message TfLite {}
// Delegate to run GPU inference depending on the device.
// (Can use OpenGl, OpenCl, Metal depending on the device.)
message Gpu {}
// Android only.
message Nnapi {}
oneof delegate {
TfLite tflite = 1;
Gpu gpu = 2;
Nnapi nnapi = 3;
}
}
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
// On mobile, this is generally just modelname.tflite.
optional string model_path = 1;
@@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions {
// Whether the TF Lite GPU or CPU backend should be used. Effective only when
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
// used.
optional bool use_gpu = 2 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_gpu = 2 [deprecated = true, default = false];
// Android only. When true, an NNAPI delegate will be used for inference.
// If NNAPI is not available, then the default CPU delegate will be used
// automatically.
optional bool use_nnapi = 3 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_nnapi = 3 [deprecated = true, default = false];
// The number of threads available to the interpreter. Effective only when
// input tensors are on CPU and 'use_gpu' is false.
optional int32 cpu_num_thread = 4 [default = -1];
// TfLite delegate to run inference.
// NOTE: calculator is free to choose delegate if not specified explicitly.
// NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
// precedence over use_* deprecated options.)
optional Delegate delegate = 5;
}
@@ -16,6 +16,8 @@
#include <string>
#include <vector>
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -39,13 +41,7 @@ namespace mediapipe {
using ::tflite::Interpreter;
class TfLiteInferenceCalculatorTest : public ::testing::Test {
protected:
std::unique_ptr<CalculatorRunner> runner_ = nullptr;
};
// Tests a simple add model that adds an input tensor to itself.
TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
void DoSmokeTest(absl::string_view delegate) {
const int width = 8;
const int height = 8;
const int channels = 3;
@@ -73,23 +69,24 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
auto input_vec = absl::make_unique<std::vector<TfLiteTensor>>();
input_vec->emplace_back(*tensor);
std::string graph_proto = R"(
input_stream: "tensor_in"
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:tensor_in"
output_stream: "TENSORS:tensor_out"
options {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
$delegate
}
}
}
)";
ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1);
// Prepare single calculator graph to and wait for packets.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tensor_in"
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:tensor_in"
output_stream: "TENSORS:tensor_out"
options {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
use_gpu: false
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
}
}
}
)");
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
CalculatorGraph graph(graph_config);
@@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
MP_ASSERT_OK(graph.WaitUntilDone());
}
// Tests a simple add model that adds an input tensor to itself.
TEST(TfLiteInferenceCalculatorTest, SmokeTest) {
DoSmokeTest(/*delegate=*/"");
DoSmokeTest(/*delegate=*/"delegate { tflite {} }");
}
} // namespace mediapipe
@@ -28,6 +28,21 @@ namespace mediapipe {
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
// tensor will be used. The size of the values must be
// (num_dimension x num_landmarks).
//
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides corresponding side packet and/or field in the calculator
// options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides corresponding side packet and/or field in the calculator options.
//
// Input side packet:
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides the corresponding field in the calculator options.
//
// Output:
// LANDMARKS(optional) - Result MediaPipe landmarks.
// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
@@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase {
private:
::mediapipe::Status LoadOptions(CalculatorContext* cc);
int num_landmarks_ = 0;
bool flip_vertically_ = false;
bool flip_horizontally_ = false;
::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_;
};
@@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LANDMARKS")) {
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
}
@@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
<< "Must provide input with/height for getting normalized landmarks.";
}
if (cc->Outputs().HasTag("LANDMARKS") &&
(options_.flip_vertically() || options_.flip_horizontally())) {
(options_.flip_vertically() || options_.flip_horizontally() ||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input with/height for using flip_vertically option "
"when outputing landmarks in absolute coordinates.";
}
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
: options_.flip_horizontally();
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
: options_.flip_vertically();
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
return ::mediapipe::OkStatus();
}
@@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
const int offset = ld * num_dimensions;
Landmark* landmark = output_landmarks.add_landmark();
if (options_.flip_horizontally()) {
if (flip_horizontally_) {
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
} else {
landmark->set_x(raw_landmarks[offset]);
}
if (num_dimensions > 1) {
if (options_.flip_vertically()) {
if (flip_vertically_) {
landmark->set_y(options_.input_image_height() -
raw_landmarks[offset + 1]);
} else {
+63
View File
@@ -39,6 +39,15 @@ proto_library(
],
)
proto_library(
name = "timed_box_list_id_to_label_calculator_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
name = "latency_proto",
srcs = ["latency.proto"],
@@ -113,6 +122,18 @@ mediapipe_cc_proto_library(
],
)
mediapipe_cc_proto_library(
name = "timed_box_list_id_to_label_calculator_cc_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [
":timed_box_list_id_to_label_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "latency_cc_proto",
srcs = ["latency.proto"],
@@ -313,6 +334,34 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "timed_box_list_id_to_label_calculator",
srcs = ["timed_box_list_id_to_label_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":timed_box_list_id_to_label_calculator_cc_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/util/tracking:box_tracker_cc_proto",
"//mediapipe/util:resource_util",
] + select({
"//mediapipe:android": [
"//mediapipe/util/android/file/base",
],
"//mediapipe:apple": [
"//mediapipe/util/android/file/base",
],
"//mediapipe:macos": [
"//mediapipe/framework/port:file_helpers",
],
"//conditions:default": [
"//mediapipe/framework/port:file_helpers",
],
}),
alwayslink = 1,
)
cc_library(
name = "non_max_suppression_calculator",
srcs = ["non_max_suppression_calculator.cc"],
@@ -437,6 +486,7 @@ cc_library(
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/types:optional",
],
alwayslink = 1,
)
@@ -928,6 +978,19 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "local_file_pattern_contents_calculator",
srcs = ["local_file_pattern_contents_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "filter_collection_calculator",
srcs = ["filter_collection_calculator.cc"],
@@ -39,13 +39,13 @@ namespace mediapipe {
namespace {
constexpr char kInputFrameTag[] = "INPUT_FRAME";
constexpr char kOutputFrameTag[] = "OUTPUT_FRAME";
constexpr char kInputFrameTag[] = "IMAGE";
constexpr char kOutputFrameTag[] = "IMAGE";
constexpr char kInputVectorTag[] = "VECTOR";
constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// A calculator for rendering data on images.
//
// Inputs:
// 1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer)
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
// containing the input image.
// If output is CPU, and input isn't provided, the renderer creates a
// blank canvas with the width, height and color provided in the options.
@@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// input vector items. These input streams are tagged with "VECTOR".
//
// Output:
// 1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer).
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
//
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
// output format is the same as input except for GRAY8 where the output is in
@@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (CPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME:image_frames"
// input_stream: "IMAGE:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME:decorated_frames"
// output_stream: "IMAGE:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }
@@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (GPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME_GPU:image_frames"
// input_stream: "IMAGE_GPU:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME_GPU:decorated_frames"
// output_stream: "IMAGE_GPU:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe//framework/packet.h"
#include "mediapipe/calculators/util/detection_label_id_to_text_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/resource_util.h"
@@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
} // namespace
::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect(
const Detection& detection, Rect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
Rect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX)
<< "Only Detection with formats of BOUNDING_BOX can be converted to Rect";
@@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect(
const Detection& detection, NormalizedRect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX)
<< "Only Detection with formats of RELATIVE_BOUNDING_BOX can be "
@@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
}
std::pair<int, int> image_size;
if (rotate_) {
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
// Get dynamic calculator options (e.g. `image_size`).
const DetectionSpec detection_spec = GetDetectionSpec(cc);
if (cc->Outputs().HasTag(kRectTag)) {
auto output_rect = absl::make_unique<Rect>();
MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(
DetectionToRect(detections[0], detection_spec, output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs().Tag(kRectTag).Add(output_rect.release(),
cc->InputTimestamp());
}
if (cc->Outputs().HasTag(kNormRectTag)) {
auto output_rect = absl::make_unique<NormalizedRect>();
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec,
output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs()
.Tag(kNormRectTag)
@@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
if (cc->Outputs().HasTag(kRectsTag)) {
auto output_rects = absl::make_unique<std::vector<Rect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec,
&(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs().Tag(kRectsTag).Add(output_rects.release(),
@@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
auto output_rects =
absl::make_unique<std::vector<NormalizedRect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(
detections[i], detection_spec, &(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs()
@@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
return ::mediapipe::OkStatus();
}
float DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const std::pair<int, int> image_size) {
::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const DetectionSpec& detection_spec,
float* rotation) {
const auto& location_data = detection.location_data();
const auto& image_size = detection_spec.image_size;
RET_CHECK(image_size) << "Image size is required to calculate rotation";
const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() *
image_size.second;
image_size->second;
const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() *
image_size.second;
image_size->second;
float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0);
*rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
return NormalizeRadians(rotation);
return ::mediapipe::OkStatus();
}
DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec(
const CalculatorContext* cc) {
absl::optional<std::pair<int, int>> image_size;
if (cc->Inputs().HasTag(kImageSizeTag)) {
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
return {image_size};
}
REGISTER_CALCULATOR(DetectionsToRectsCalculator);
@@ -16,6 +16,7 @@
#include <cmath>
#include "absl/types/optional.h"
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@@ -27,6 +28,13 @@
namespace mediapipe {
// Dynamic options passed as calculator `input_stream` that can be used for
// calculation of rectangle or rotation for given detection. Does not include
// static calculator options which are available via private fields.
struct DetectionSpec {
absl::optional<std::pair<int, int>> image_size;
};
// A calculator that converts Detection proto to Rect proto.
//
// Detection is the format for encoding one or more detections in an image.
@@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override;
protected:
virtual float ComputeRotation(const ::mediapipe::Detection& detection,
const std::pair<int, int> image_size);
virtual ::mediapipe::Status DetectionToRect(
const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect);
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, ::mediapipe::Rect* rect);
virtual ::mediapipe::Status DetectionToNormalizedRect(
const ::mediapipe::Detection& detection,
::mediapipe::NormalizedRect* rect);
const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect);
virtual ::mediapipe::Status ComputeRotation(
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, float* rotation);
virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc);
static inline float NormalizeRadians(float angle) {
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
@@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:adjusted_landmarks"
// }
//
// node {
// calculator: "LandmarkLetterboxRemovalCalculator"
// input_stream: "LANDMARKS:0:landmarks_0"
// input_stream: "LANDMARKS:1:landmarks_1"
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:0:adjusted_landmarks_0"
// output_stream: "LANDMARKS:1:adjusted_landmarks_1"
// }
class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
cc->Inputs().HasTag(kLetterboxPaddingTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@@ -89,38 +94,45 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& letterbox_padding =
cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
const float left = letterbox_padding[0];
const float top = letterbox_padding[1];
const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
const NormalizedLandmarkList& input_landmarks =
input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
}
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
@@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT";
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:projected_landmarks"
// }
//
// node {
// calculator: "LandmarkProjectionCalculator"
// input_stream: "NORM_LANDMARKS:0:landmarks_0"
// input_stream: "NORM_LANDMARKS:1:landmarks_1"
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0"
// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1"
// }
class LandmarkProjectionCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase {
cc->Inputs().HasTag(kRectTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@@ -85,41 +90,50 @@ class LandmarkProjectionCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kRectTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
const float angle = options.ignore_rotation() ? 0 : input_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
new_x = new_x * input_rect.width() + input_rect.x_center();
new_y = new_y * input_rect.height() + input_rect.y_center();
const auto& input_landmarks = input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
const float angle =
options.ignore_rotation() ? 0 : input_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
new_x = new_x * input_rect.width() + input_rect.x_center();
new_y = new_y * input_rect.height() + input_rect.y_center();
new_landmark->set_x(new_x);
new_landmark->set_y(new_y);
// Keep z-coord as is.
new_landmark->set_z(landmark.z());
}
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
};
@@ -0,0 +1,75 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// The calculator takes the path to local directory and desired file suffix to
// mach as input side packets, and outputs the contents of those files that
// match the pattern. Those matched files will be sent sequentially through the
// output stream with incremental timestamp difference by 1.
//
// Example config:
// node {
// calculator: "LocalFilePatternContentsCalculator"
// input_side_packet: "FILE_DIRECTORY:file_directory"
// input_side_packet: "FILE_SUFFIX:file_suffix"
// output_stream: "CONTENTS:contents"
// }
class LocalFilePatternContentsCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Tag("FILE_DIRECTORY").Set<std::string>();
cc->InputSidePackets().Tag("FILE_SUFFIX").Set<std::string>();
cc->Outputs().Tag("CONTENTS").Set<std::string>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory(
cc->InputSidePackets().Tag("FILE_DIRECTORY").Get<std::string>(),
cc->InputSidePackets().Tag("FILE_SUFFIX").Get<std::string>(),
&filenames_));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (current_output_ < filenames_.size()) {
auto contents = absl::make_unique<std::string>();
LOG(INFO) << filenames_[current_output_];
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
filenames_[current_output_], contents.get()));
++current_output_;
cc->Outputs()
.Tag("CONTENTS")
.Add(contents.release(), Timestamp(current_output_));
} else {
return tool::StatusStop();
}
return ::mediapipe::OkStatus();
}
private:
std::vector<std::string> filenames_;
int current_output_ = 0;
};
REGISTER_CALCULATOR(LocalFilePatternContentsCalculator);
} // namespace mediapipe
@@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect(
void SetRect(bool normalized, double xmin, double ymin, double width,
double height, double rotation,
RenderAnnotation::Rectangle* rect) {
if (xmin + width < 0.0 || ymin + height < 0.0) return;
if (normalized) {
if (xmin > 1.0 || ymin > 1.0) return;
if (rotation == 0.0) {
if (xmin + width < 0.0 || ymin + height < 0.0) return;
if (normalized) {
if (xmin > 1.0 || ymin > 1.0) return;
}
}
rect->set_normalized(normalized);
rect->set_left(normalized ? std::max(xmin, 0.0) : xmin);
rect->set_top(normalized ? std::max(ymin, 0.0) : ymin);
rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width);
rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height);
rect->set_left(xmin);
rect->set_top(ymin);
rect->set_right(xmin + width);
rect->set_bottom(ymin + height);
rect->set_rotation(rotation);
}
@@ -0,0 +1,105 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/timed_box_list_id_to_label_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/resource_util.h"
#include "mediapipe/util/tracking/box_tracker.pb.h"
#if defined(MEDIAPIPE_MOBILE)
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/helpers.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
#endif
namespace mediapipe {
using mediapipe::TimedBoxProto;
using mediapipe::TimedBoxProtoList;
// Takes a label map (from label IDs to names), and populate the label field in
// TimedBoxProto according to it's ID.
//
// Example usage:
// node {
// calculator: "TimedBoxListIdToLabelCalculator"
// input_stream: "input_timed_box_list"
// output_stream: "output_timed_box_list"
// node_options: {
// [mediapipe.TimedBoxListIdToLabelCalculatorOptions] {
// label_map_path: "labelmap.txt"
// }
// }
// }
class TimedBoxListIdToLabelCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
std::unordered_map<int, std::string> label_map_;
};
REGISTER_CALCULATOR(TimedBoxListIdToLabelCalculator);
::mediapipe::Status TimedBoxListIdToLabelCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<TimedBoxProtoList>();
cc->Outputs().Index(0).Set<TimedBoxProtoList>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status TimedBoxListIdToLabelCalculator::Open(
CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
const auto& options =
cc->Options<::mediapipe::TimedBoxListIdToLabelCalculatorOptions>();
std::string string_path;
ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path()));
std::string label_map_string;
MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
std::istringstream stream(label_map_string);
std::string line;
int i = 0;
while (std::getline(stream, line)) {
label_map_[i++] = line;
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status TimedBoxListIdToLabelCalculator::Process(
CalculatorContext* cc) {
const auto& input_list = cc->Inputs().Index(0).Get<TimedBoxProtoList>();
auto output_list = absl::make_unique<TimedBoxProtoList>();
for (const auto& input_box : input_list.box()) {
TimedBoxProto* box_ptr = output_list->add_box();
*box_ptr = input_box;
if (label_map_.find(input_box.id()) != label_map_.end()) {
box_ptr->set_label(label_map_[input_box.id()]);
}
}
cc->Outputs().Index(0).Add(output_list.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
@@ -0,0 +1,28 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message TimedBoxListIdToLabelCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional TimedBoxListIdToLabelCalculatorOptions ext = 297701606;
}
// Path to a label map file for getting the actual name of detected classes.
optional string label_map_path = 1;
}
@@ -66,6 +66,25 @@ void AddTimedBoxProtoToRenderData(
rect->set_bottom(box_proto.bottom());
rect->set_rotation(box_proto.rotation());
}
if (box_proto.has_label()) {
auto* label_annotation = render_data->add_render_annotations();
label_annotation->mutable_color()->set_r(options.box_color().r());
label_annotation->mutable_color()->set_g(options.box_color().g());
label_annotation->mutable_color()->set_b(options.box_color().b());
label_annotation->set_thickness(options.thickness());
RenderAnnotation::Text* text = label_annotation->mutable_text();
text->set_display_text(box_proto.label());
text->set_normalized(true);
constexpr float text_left_start = 0.3f;
text->set_left((1.0f - text_left_start) * box_proto.left() +
text_left_start * box_proto.right());
constexpr float text_baseline = 0.6f;
text->set_baseline(text_baseline * box_proto.bottom() +
(1.0f - text_baseline) * box_proto.top());
constexpr float text_height = 0.2f;
text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height);
}
}
} // namespace
+1
View File
@@ -368,6 +368,7 @@ cc_test(
cc_test(
name = "tvl1_optical_flow_calculator_test",
srcs = ["tvl1_optical_flow_calculator_test.cc"],
linkstatic = 1,
deps = [
":tvl1_optical_flow_calculator",
"//mediapipe/framework:calculator_framework",
+3
View File
@@ -15,6 +15,9 @@ For overall context on AutoFlip, please read this
Run the following command to build the AutoFlip pipeline:
Note: AutoFlip currently only works with OpenCV 3 . Please verify your OpenCV
version beforehand.
```bash
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/autoflip:run_autoflip
```
+8
View File
@@ -44,6 +44,14 @@ graphs can be easily adapted to run on CPU v.s. GPU.
[Object Detection and Tracking with GPU](./object_tracking_mobile_gpu.md) illustrates how to
use MediaPipe for object detection and tracking.
### Objectron: 3D Object Detection and Tracking with GPU
[MediaPipe Objectron is 3D Object Detection with GPU](./objectron_mobile_gpu.md)
illustrates mobile real-time 3D object detection and tracking pipeline for every
day objects like shoes and chairs
* [Android](./objectron_mobile_gpu.md)
### Face Detection with GPU
[Face Detection with GPU](./face_detection_mobile_gpu.md) illustrates how to use
+2 -2
View File
@@ -259,9 +259,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
```
+2 -2
View File
@@ -229,9 +229,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video_cpu"
input_stream: "IMAGE:input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of
+2 -2
View File
@@ -221,8 +221,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```
+2 -2
View File
@@ -136,10 +136,10 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "detection_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```
+2 -2
View File
@@ -716,10 +716,10 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "landmark_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```
Binary file not shown.

After

Width:  |  Height:  |  Size: 100 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 113 KiB

+16 -14
View File
@@ -40,7 +40,7 @@ To build and run iOS apps:
$ cd mediapipe
```
2. Install Bazel (version between 0.24.1 and 1.2.1).
2. Install Bazel (version between 1.0.0 and 1.2.1).
Follow the official
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
@@ -152,7 +152,7 @@ To build and run iOS apps:
$ cd mediapipe
```
2. Install Bazel (version between 0.24.1 and 1.2.1).
2. Install Bazel (version between 1.0.0 and 1.2.1).
Follow the official
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
@@ -241,7 +241,7 @@ To build and run iOS apps:
$ cd mediapipe
```
3. Install Bazel (version between 0.24.1 and 1.1.0).
3. Install Bazel (version between 1.0.0 and 1.1.0).
Option 1. Use package manager tool to install Bazel 1.1.0
@@ -364,8 +364,10 @@ To build and run iOS apps:
### Installing on Windows Subsystem for Linux (WSL)
Note: WSL has historically not provided access to USB cameras. Mediapipe can use
a video file as input.
Note: The pre-built OpenCV packages don't support cameras in WSL. Unless you
[compile](https://funvision.blogspot.com/2019/12/opencv-web-camera-and-video-streams-in.html)
OpenCV with FFMPEG and GStreamer in WSL, the live demos won't work with any
cameras. Alternatively, you use a video file as input.
1. Follow the
[instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
@@ -373,7 +375,7 @@ a video file as input.
2. Install Windows ADB and start the ADB server in Windows.
Note: Windows and WSLs adb versions must be the same version, e.g., if WSL
Note: Windows' and WSLs adb versions must be the same version, e.g., if WSL
has ADB 1.0.39, you need to download the corresponding Windows ADB from
[here](https://dl.google.com/android/repository/platform-tools_r26.0.1-windows.zip).
@@ -389,18 +391,18 @@ a video file as input.
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
```
5. Install Bazel (version between 0.24.1 and 1.2.1).
5. Install Bazel (version between 1.0.0 and 1.2.1).
```bash
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
https://storage.googleapis.com/bazel/0.27.0/release/bazel-0.27.0-installer-linux-x86_64.sh && \
sudo mkdir -p /usr/local/bazel/0.27.0 && \
chmod 755 bazel-0.27.0-installer-linux-x86_64.sh && \
sudo ./bazel-0.27.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/0.27.0 && \
source /usr/local/bazel/0.27.0/lib/bazel/bin/bazel-complete.bash
https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
sudo mkdir -p /usr/local/bazel/1.0.0 && \
chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/0.27.0/lib/bazel/bin/bazel version && \
alias bazel='/usr/local/bazel/0.27.0/lib/bazel/bin/bazel'
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
```
6. Checkout MediaPipe repository.
@@ -745,11 +745,11 @@ node {
# a vector of RenderData objects and draws each of them on the input frame.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "multi_hand_rects_render_data"
input_stream: "multi_palm_rects_render_data"
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```
+5 -4
View File
@@ -26,6 +26,7 @@ To build and run the TensorFlow example on desktop, run:
$ bazel build -c opt \
--define MEDIAPIPE_DISABLE_GPU=1 \
--define no_aws_support=true \
--linkopt=-s \
mediapipe/examples/desktop/object_detection:object_detection_tensorflow
# It should print:
@@ -163,9 +164,9 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified
@@ -396,9 +397,9 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified
@@ -230,9 +230,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video_cpu"
input_stream: "IMAGE:throttled_input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of
@@ -212,8 +212,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```
+4 -3
View File
@@ -467,9 +467,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detections_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```
@@ -484,7 +484,8 @@ CPU.
To build and run the app:
```bash
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu \
--define MEDIAPIPE_DISABLE_GPU=1
bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
--calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt
+489
View File
@@ -0,0 +1,489 @@
# MediaPipe Objectron (GPU)
This doc focuses on the
[below example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
that performs 3D object detection and tracking with TensorFlow Lite on GPU.
Objectron for shoes | Objectron for chairs
:-----------------------------------------------------------------------------: | :------------------:
![objectron_shoe_android_gpu_gif](images/mobile/objectron_shoe_android_gpu.gif) | ![objectron_chair_android_gpu_gif](images/mobile/objectron_chair_android_gpu.gif)
For overall context on MediaPipe Objectron, please read the
[Google AI Blog](https://mediapipe.page.link/objectron-aiblog). The Objectron's
ML model (see also the [model card](https://mediapipe.page.link/objectron-mc))
estimates a 3D bounding box for the detected object.
## Android
[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d)
An arm64 build of Objectron for shoes can be
[downloaded here](https://drive.google.com/open?id=1S0K4hbWt3o31FfQ4QU3Rz7IHrvOUMx1d),
and for chairs can be
[downloaded here](https://drive.google.com/open?id=1MM8K-13bXLCVS1EHQ-KgkVyEahEPrKej).
To build and install the Objectron for shoes:
```bash
bazel build -c opt --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
```
Similarly to build and install the Objectron for chairs, add **--define
chair=true** flag to build command.
```bash
bazel build -c opt --define chair=true --config android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d:objectdetection3d
```
Once the app is built, install in on Android device with:
```bash
adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/objectdetection3d.apk
```
## Graph
The Objectron main graph internally utilizes the Objectron detection subgraph,
and the Objectron tracking subgraph. To visualize the graph as shown above, copy
the text specification of the graph below and paste it into
[MediaPipe Visualizer](https://viz.mediapipe.dev/).
### Main Graph
This is the main graph for the shoe detector. This graph runs detection and
tracking and renders the output to the display.
![object_detection_mobile_gpu_graph](images/mobile/object_detection_3d_android_gpu.png)
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/shoe_classic_occlusion_tracking.pbtxt)
```bash
# MediaPipe object detection 3D with tracking graph.
# Images on GPU coming into and out of the graph.
input_stream: "input_video"
output_stream: "output_video"
# Creates a copy of the input_video stream. At the end of the graph, the
# GlAnimationOverlayCalculator will consume the input_video texture and draws
# on top of it.
node: {
calculator: "GlScalerCalculator"
input_stream: "VIDEO:input_video"
output_stream: "VIDEO:input_video_copy"
}
# Resamples the images by specific frame rate. This calculator is used to
# control the frequecy of subsequent calculators/subgraphs, e.g. less power
# consumption for expensive process.
node {
calculator: "PacketResamplerCalculator"
input_stream: "DATA:input_video_copy"
output_stream: "DATA:sampled_input_video"
node_options: {
[type.googleapis.com/mediapipe.PacketResamplerCalculatorOptions] {
frame_rate: 5
}
}
}
node {
calculator: "ObjectronDetectionSubgraphGpu"
input_stream: "IMAGE_GPU:sampled_input_video"
output_stream: "ANNOTATIONS:objects"
}
node {
calculator: "ObjectronTrackingSubgraphGpu"
input_stream: "FRAME_ANNOTATION:objects"
input_stream: "IMAGE_GPU:input_video_copy"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
}
# The rendering nodes:
# We are rendering two meshes: 1) a 3D bounding box, which we overlay directly
# on the texture, and 2) a shoe CAD model, which we use as an occlusion mask.
# These models are designed using different tools, so we supply a transformation
# to bring both of them to the Objectron's coordinate system.
# Creates a model matrices for the tracked object given the lifted 3D points.
# This calculator does two things: 1) Estimates object's pose (orientation,
# translation, and scale) from the 3D vertices, and
# 2) bring the object from the objectron's coordinate system to the renderer
# (OpenGL) coordinate system. Since the final goal is to render a mesh file on
# top of the object, we also supply a transformation to bring the mesh to the
# objectron's coordinate system, and rescale mesh to the unit size.
node {
calculator: "AnnotationsToModelMatricesCalculator"
input_stream: "ANNOTATIONS:lifted_tracked_objects"
output_stream: "MODEL_MATRICES:model_matrices"
node_options: {
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
# Re-scale the CAD model to the size of a unit box
model_scale: 0.05
model_scale: 0.05
model_scale: 0.05
# Bring the box CAD model to objectron's coordinate system. This
# is equivalent of -pi/2 rotation along the y-axis (right-hand rule):
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitY())
model_transformation: 0.0
model_transformation: 0.0
model_transformation: -1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 1.0
}
}
}
# Compute the model matrices for the CAD model of the shoe, to be used as an
# occlusion mask. The model will be rendered at the exact same location as the
# bounding box.
node {
calculator: "AnnotationsToModelMatricesCalculator"
input_stream: "ANNOTATIONS:lifted_tracked_objects"
output_stream: "MODEL_MATRICES:mask_model_matrices"
#input_side_packet: "MODEL_SCALE:model_scale"
node_options: {
[type.googleapis.com/mediapipe.AnnotationsToModelMatricesCalculatorOptions] {
# Re-scale the CAD model to the size of a unit box
model_scale: 0.45
model_scale: 0.25
model_scale: 0.15
# Bring the shoe CAD model to Deep Pursuit 3D's coordinate system. This
# is equivalent of -pi/2 rotation along the x-axis (right-hand rule):
# Eigen::AngleAxisf(-M_PI / 2., Eigen::Vector3f::UnitX())
model_transformation: 1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: -1.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 0.0
model_transformation: 1.0
}
}
}
# Render everything together. First we render the 3D bounding box animation,
# then we render the occlusion mask.
node: {
calculator: "GlAnimationOverlayCalculator"
input_stream: "VIDEO:input_video"
input_stream: "MODEL_MATRICES:model_matrices"
input_stream: "MASK_MODEL_MATRICES:mask_model_matrices"
output_stream: "output_video"
input_side_packet: "TEXTURE:box_texture"
input_side_packet: "ANIMATION_ASSET:box_asset_name"
input_side_packet: "MASK_TEXTURE:obj_texture"
input_side_packet: "MASK_ASSET:obj_asset_name"
node_options: {
[type.googleapis.com/mediapipe.GlAnimationOverlayCalculatorOptions] {
# Output resolution is 480x640 with the aspect ratio of 0.75
aspect_ratio: 0.75
vertical_fov_degrees: 70.
animation_speed_fps: 25
}
}
}
```
### Objectron Detection Subgraph
Objectron detection subgraph uses the *TfLiteInferenceCalculator* to run
inference and decodes the output tensor to *FrameAnnotation* protobuf. The
*FrameAnnotation* contains nine keypoints: the bounding box's center, as well as
its eight vertices. The boxes will be passed to the Objectron tracking subgraph.
![object_detection_subgraph](images/mobile/objectron_detection_subgraph.png)
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_detection_gpu.pbtxt)
```bash
# MediaPipe Objectron detection gpu subgraph
type: "ObjectronDetectionSubgraphGpu"
input_stream: "IMAGE_GPU:input_video"
output_stream: "ANNOTATIONS:objects"
# Transforms the input image on GPU to a 480x640 image. To scale the input
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
# resulting in potential letterboxing in the transformed image.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "IMAGE_GPU:transformed_input_video"
node_options: {
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
output_width: 480
output_height: 640
scale_mode: FIT
}
}
}
# Converts the transformed input image on GPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE_GPU:transformed_input_video"
output_stream: "TENSORS_GPU:image_tensor"
}
# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
calculator: "TfLiteCustomOpResolverCalculator"
output_side_packet: "opresolver"
node_options: {
[type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
use_gpu: true
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS_GPU:image_tensor"
output_stream: "TENSORS:detection_tensors"
input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
node_options: {
[type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
model_path: "object_detection_3d.tflite"
}
}
}
# Decodes the model's output tensor (the heatmap and the distance fields) to 2D
# keypoints. There are nine 2D keypoints: one center keypoint and eight vertices
# for the 3D bounding box. The calculator parameters determine's the decoder's
# sensitivity.
node {
calculator: "TfLiteTensorsToObjectsCalculator"
input_stream: "TENSORS:detection_tensors"
output_stream: "ANNOTATIONS:objects"
node_options: {
[type.googleapis.com/mediapipe.TfLiteTensorsToObjectsCalculatorOptions] {
num_classes: 1
num_keypoints: 9
decoder_config {
heatmap_threshold: 0.6
local_max_distance: 2
offset_scale_coef: 1.0
voting_radius: 2
voting_allowance: 1
voting_threshold: 0.2
}
}
}
}
```
### Object Tracking Subgraph
Object tracking subgraph uses a *BoxTracker* calculator which is a generic
tracking library, also used in
[Mediapipe's 2D Object Detection and Tracking](https://github.com/google/mediapipe/tree/master/mediapipe/g3doc/object_tracking_mobile_gpu.md).
The tracking runs every frame and when a new detection is available, it
consolidates the detection and tracking results. The tracker tracks the box with
its 2D keypoints, so at the end we lift the 2D keypoints to 3D using EPnP
algorithm in *Lift2DFrameAnnotationTo3D* Calculator.
![object_tracking_subgraph](images/mobile/objectron_tracking_subgraph.png)
[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/subgraphs/objectron_tracking_gpu.pbtxt)
```bash
# MediaPipe Objectron tracking gpu subgraph
type: "ObjectronTrackingSubgraphGpu"
input_stream: "FRAME_ANNOTATION:objects"
input_stream: "IMAGE_GPU:input_video"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
# Converts the detected keypoints to Boxes, used by the tracking subgraph.
node {
calculator: "FrameAnnotationToTimedBoxListCalculator"
input_stream: "FRAME_ANNOTATION:objects"
output_stream: "BOXES:start_pos"
}
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "IMAGE_GPU:downscaled_input_video"
node_options: {
[type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
output_width: 240
output_height: 320
}
}
}
# Converts GPU buffer to ImageFrame for processing tracking.
node: {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "downscaled_input_video"
output_stream: "downscaled_input_video_cpu"
}
# Performs motion analysis on an incoming video stream.
node: {
calculator: "MotionAnalysisCalculator"
input_stream: "VIDEO:downscaled_input_video_cpu"
output_stream: "CAMERA:camera_motion"
output_stream: "FLOW:region_flow"
node_options: {
[type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: {
analysis_options {
analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
flow_options {
fast_estimation_min_block_size: 100
top_inlier_sets: 1
frac_inlier_error_threshold: 3e-3
downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
verification_distance: 5.0
verify_long_feature_acceleration: true
verify_long_feature_trigger_ratio: 0.1
tracking_options {
max_features: 500
adaptive_extraction_levels: 2
min_eig_val_settings {
adaptive_lowest_quality_level: 2e-4
}
klt_tracker_implementation: KLT_OPENCV
}
}
}
}
}
}
# Reads optical flow fields defined in
# mediapipe/framework/formats/motion/optical_flow_field.h,
# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
# to 0-255.
node: {
calculator: "FlowPackagerCalculator"
input_stream: "FLOW:region_flow"
input_stream: "CAMERA:camera_motion"
output_stream: "TRACKING:tracking_data"
node_options: {
[type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: {
flow_packager_options: {
binary_tracking_data_support: false
}
}
}
}
# Tracks box positions over time.
node: {
calculator: "BoxTrackerCalculator"
input_stream: "TRACKING:tracking_data"
input_stream: "TRACK_TIME:input_video"
input_stream: "START_POS:start_pos"
input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
input_stream_info: {
tag_index: "CANCEL_OBJECT_ID"
back_edge: true
}
output_stream: "BOXES:boxes"
input_stream_handler {
input_stream_handler: "SyncSetInputStreamHandler"
options {
[mediapipe.SyncSetInputStreamHandlerOptions.ext] {
sync_set {
tag_index: "TRACKING"
tag_index: "TRACK_TIME"
}
sync_set {
tag_index: "START_POS"
}
sync_set {
tag_index: "CANCEL_OBJECT_ID"
}
}
}
}
node_options: {
[type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: {
tracker_options: {
track_step_options {
track_object_and_camera: true
tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
inlier_spring_force: 0.0
static_motion_temporal_ratio: 3e-2
}
}
visualize_tracking_data: false
streaming_track_data_cache_size: 100
}
}
}
# Consolidates tracking and detection results.
node {
calculator: "FrameAnnotationTrackerCalculator"
input_stream: "FRAME_ANNOTATION:objects"
input_stream: "TRACKED_BOXES:boxes"
output_stream: "TRACKED_FRAME_ANNOTATION:tracked_objects"
output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
node_options: {
[type.googleapis.com/mediapipe.FrameAnnotationTrackerCalculatorOptions] {
img_width: 240
img_height: 320
}
}
input_stream_handler {
input_stream_handler: "SyncSetInputStreamHandler"
options {
[mediapipe.SyncSetInputStreamHandlerOptions.ext] {
sync_set {
tag_index: "FRAME_ANNOTATION"
}
sync_set {
tag_index: "TRACKED_BOXES"
}
}
}
}
}
# Lift the tracked 2D keypoints to 3D using EPnP algorithm.
node {
calculator: "Lift2DFrameAnnotationTo3DCalculator"
input_stream: "FRAME_ANNOTATION:tracked_objects"
output_stream: "LIFTED_FRAME_ANNOTATION:lifted_tracked_objects"
}
```
+6 -4
View File
@@ -61,6 +61,8 @@ videos.
```bash
# cd to the root directory of the MediaPipe repo
cd -
pip3 install tf_slim
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
```
@@ -78,7 +80,7 @@ videos.
5. Run the MediaPipe binary to extract the features.
```bash
bazel build -c opt \
bazel build -c opt --linkopt=-s \
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
@@ -126,13 +128,13 @@ the inference for both local videos and the dataset
2. Build the inference binary.
```bash
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
mediapipe/examples/desktop/youtube8m:model_inference
```
3. Run the python web server.
Note: pip install absl-py
Note: pip3 install absl-py
```bash
python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
@@ -162,7 +164,7 @@ the inference for both local videos and the dataset
3. Build and run the inference binary.
```bash
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
mediapipe/examples/desktop/youtube8m:model_inference
# segment_size is the number of seconds window of frames.
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.google.mediapipe.apps.objectdetection3d">
<uses-sdk
android:minSdkVersion="21"
android:targetSdkVersion="27" />
<!-- For using the camera -->
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<uses-feature android:name="android.hardware.camera.autofocus" />
<!-- For MediaPipe -->
<uses-feature android:glEsVersion="0x00020000" android:required="true" />
<application
android:allowBackup="true"
android:label="@string/app_name"
android:supportsRtl="true"
android:theme="@style/AppTheme">
<activity
android:name=".MainActivity"
android:exported="true"
android:screenOrientation="portrait">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
@@ -0,0 +1,115 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:private"])
cc_binary(
name = "libmediapipe_jni.so",
linkshared = 1,
linkstatic = 1,
deps = [
"//mediapipe/graphs/object_detection_3d:mobile_calculators",
"//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
],
)
cc_library(
name = "mediapipe_jni_lib",
srcs = [":libmediapipe_jni.so"],
alwayslink = 1,
)
# To use the "chair" model instead of the default "shoes" model,
# add "--define chair=true" to the bazel build command.
config_setting(
name = "use_chair_model",
define_values = {
"chair": "true",
},
)
# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
# easily incorporated into the app via, for example,
# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
genrule(
name = "binary_graph",
srcs = select({
"//conditions:default": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_shoe"],
":use_chair_model": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph_chair"],
}),
outs = ["objectdetection3d.binarypb"],
cmd = "cp $< $@",
)
genrule(
name = "model",
srcs = select({
"//conditions:default": ["//mediapipe/models:object_detection_3d_sneakers.tflite"],
":use_chair_model": ["//mediapipe/models:object_detection_3d_chair.tflite"],
}),
outs = ["object_detection_3d.tflite"],
cmd = "cp $< $@",
)
android_library(
name = "mediapipe_lib",
srcs = glob(["*.java"]),
assets = [
":binary_graph",
":model",
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:box.obj.uuu",
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets:classic_colors.png",
] + select({
"//conditions:default": [
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:model.obj.uuu",
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker:texture.bmp",
],
":use_chair_model": [
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:model.obj.uuu",
"//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair:texture.bmp",
],
}),
assets_dir = "",
manifest = "AndroidManifest.xml",
resource_files = glob(["res/**"]),
deps = [
":mediapipe_jni_lib",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
"//mediapipe/java/com/google/mediapipe/components:android_components",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/java/com/google/mediapipe/glutil",
"//third_party:androidx_appcompat",
"//third_party:androidx_constraint_layout",
"//third_party:androidx_legacy_support_v4",
"//third_party:androidx_recyclerview",
"//third_party:opencv",
"@androidx_concurrent_futures//jar",
"@androidx_lifecycle//jar",
"@com_google_code_findbugs//jar",
"@com_google_guava_android//jar",
],
)
android_binary(
name = "objectdetection3d",
manifest = "AndroidManifest.xml",
manifest_values = {"applicationId": "com.google.mediapipe.apps.objectdetection3d"},
multidex = "native",
deps = [
":mediapipe_lib",
],
)
@@ -0,0 +1,280 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.apps.objectdetection3d;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.SurfaceTexture;
import android.os.Bundle;
import androidx.appcompat.app.AppCompatActivity;
import android.util.Log;
import android.util.Size;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;
import com.google.mediapipe.components.CameraHelper;
import com.google.mediapipe.components.CameraXPreviewHelper;
import com.google.mediapipe.components.ExternalTextureConverter;
import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.components.PermissionHelper;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.framework.AndroidPacketCreator;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.glutil.EglManager;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
/** Main activity of MediaPipe example apps. */
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
private static final String BINARY_GRAPH_NAME = "objectdetection3d.binarypb";
private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
private static final String OBJ_TEXTURE = "texture.bmp";
private static final String OBJ_FILE = "model.obj.uuu";
private static final String BOX_TEXTURE = "classic_colors.png";
private static final String BOX_FILE = "box.obj.uuu";
private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
// Flips the camera-preview frames vertically before sending them into FrameProcessor to be
// processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
// This is needed because OpenGL represents images assuming the image origin is at the bottom-left
// corner, whereas MediaPipe in general assumes the image origin is at top-left.
private static final boolean FLIP_FRAMES_VERTICALLY = true;
// Target resolution should be 4:3 for this application, as expected by the model and tracker.
private static final Size TARGET_RESOLUTION = new Size(1280, 960);
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.
private SurfaceTexture previewFrameTexture;
// {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
private SurfaceView previewDisplayView;
// Creates and manages an {@link EGLContext}.
private EglManager eglManager;
// Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
// frames onto a {@link Surface}.
private FrameProcessor processor;
// Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
// consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
private ExternalTextureConverter converter;
// Handles camera access via the {@link CameraX} Jetpack support library.
private CameraXPreviewHelper cameraHelper;
// Assets.
private Bitmap objTexture = null;
private Bitmap boxTexture = null;
Size cameraImageSize;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);
eglManager = new EglManager(null);
processor =
new FrameProcessor(
this,
eglManager.getNativeContext(),
BINARY_GRAPH_NAME,
INPUT_VIDEO_STREAM_NAME,
OUTPUT_VIDEO_STREAM_NAME);
processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
prepareDemoAssets();
AndroidPacketCreator packetCreator = processor.getPacketCreator();
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put("obj_asset_name", packetCreator.createString(OBJ_FILE));
inputSidePackets.put("box_asset_name", packetCreator.createString(BOX_FILE));
inputSidePackets.put("obj_texture", packetCreator.createRgbaImageFrame(objTexture));
inputSidePackets.put("box_texture", packetCreator.createRgbaImageFrame(boxTexture));
processor.setInputSidePackets(inputSidePackets);
PermissionHelper.checkAndRequestCameraPermissions(this);
}
@Override
protected void onResume() {
super.onResume();
converter = new ExternalTextureConverter(eglManager.getContext());
converter.setFlipY(FLIP_FRAMES_VERTICALLY);
converter.setConsumer(processor);
if (PermissionHelper.cameraPermissionsGranted(this)) {
startCamera();
}
}
@Override
protected void onPause() {
super.onPause();
converter.close();
}
@Override
public void onRequestPermissionsResult(
int requestCode, String[] permissions, int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
}
private void setupPreviewDisplayView() {
previewDisplayView.setVisibility(View.GONE);
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
viewGroup.addView(previewDisplayView);
previewDisplayView
.getHolder()
.addCallback(
new SurfaceHolder.Callback() {
@Override
public void surfaceCreated(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
// (Re-)Compute the ideal size of the camera-preview display (the area that the
// camera-preview frames get rendered onto, potentially with scaling and rotation)
// based on the size of the SurfaceView that contains the display.
Size viewSize = new Size(height, height * 3 / 4); // Prefer 3:4 aspect ratio.
Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
boolean isCameraRotated = cameraHelper.isCameraRotated();
cameraImageSize = cameraHelper.getFrameSize();
// Connect the converter to the camera-preview frames as its input (via
// previewFrameTexture), and configure the output width and height as the computed
// display size.
converter.setSurfaceTextureAndAttachToGLContext(
previewFrameTexture,
isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
processor.setOnWillAddFrameListener(
(timestamp) -> {
try {
int cameraTextureWidth =
isCameraRotated
? cameraImageSize.getHeight()
: cameraImageSize.getWidth();
int cameraTextureHeight =
isCameraRotated
? cameraImageSize.getWidth()
: cameraImageSize.getHeight();
// Find limiting side and scale to 3:4 aspect ratio
float aspectRatio =
(float) cameraTextureWidth / (float) cameraTextureHeight;
if (aspectRatio > 3.0 / 4.0) {
// width too big
cameraTextureWidth = (int) ((float) cameraTextureHeight * 3.0 / 4.0);
} else {
// height too big
cameraTextureHeight = (int) ((float) cameraTextureWidth * 4.0 / 3.0);
}
Packet widthPacket =
processor.getPacketCreator().createInt32(cameraTextureWidth);
Packet heightPacket =
processor.getPacketCreator().createInt32(cameraTextureHeight);
try {
processor
.getGraph()
.addPacketToInputStream("input_width", widthPacket, timestamp);
processor
.getGraph()
.addPacketToInputStream("input_height", heightPacket, timestamp);
} catch (Exception e) {
Log.e(
TAG,
"MediaPipeException encountered adding packets to width and height"
+ " input streams.");
}
widthPacket.release();
heightPacket.release();
} catch (IllegalStateException ise) {
Log.e(
TAG,
"Exception while adding packets to width and height input streams.");
}
});
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(null);
}
});
}
private void startCamera() {
cameraHelper = new CameraXPreviewHelper();
cameraHelper.setOnCameraStartedListener(
surfaceTexture -> {
previewFrameTexture = surfaceTexture;
// Make the display view visible to start showing the preview. This triggers the
// SurfaceHolder.Callback added to (the holder of) previewDisplayView.
previewDisplayView.setVisibility(View.VISIBLE);
});
cameraHelper.startCamera(
this, CAMERA_FACING, /*surfaceTexture=*/ null, /*targetSize=*/ TARGET_RESOLUTION);
cameraImageSize = cameraHelper.getFrameSize();
}
private void prepareDemoAssets() {
AndroidAssetUtil.initializeNativeAssetManager(this);
// We render from raw data with openGL, so disable decoding preprocessing
BitmapFactory.Options decodeOptions = new BitmapFactory.Options();
decodeOptions.inScaled = false;
decodeOptions.inDither = false;
decodeOptions.inPremultiplied = false;
try {
InputStream inputStream = getAssets().open(OBJ_TEXTURE);
objTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions);
inputStream.close();
} catch (Exception e) {
Log.e(TAG, "Error parsing object texture; error: " + e);
throw new IllegalStateException(e);
}
try {
InputStream inputStream = getAssets().open(BOX_TEXTURE);
boxTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions);
inputStream.close();
} catch (Exception e) {
Log.e(TAG, "Error parsing box texture; error: " + e);
throw new RuntimeException(e);
}
}
}
@@ -0,0 +1,21 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
exports_files(
srcs = glob(["**"]),
)
@@ -0,0 +1,21 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
exports_files(
srcs = glob(["**"]),
)
Binary file not shown.

After

Width:  |  Height:  |  Size: 6.9 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 410 KiB

@@ -0,0 +1,21 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
exports_files(
srcs = glob(["**"]),
)
Binary file not shown.

After

Width:  |  Height:  |  Size: 48 MiB

@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent">
<FrameLayout
android:id="@+id/preview_display_layout"
android:layout_width="fill_parent"
android:layout_height="fill_parent"
android:layout_weight="1">
<TextView
android:id="@+id/no_camera_access_view"
android:layout_height="fill_parent"
android:layout_width="fill_parent"
android:gravity="center"
android:text="@string/no_camera_access" />
</FrameLayout>
</androidx.constraintlayout.widget.ConstraintLayout>
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<color name="colorPrimary">#008577</color>
<color name="colorPrimaryDark">#00574B</color>
<color name="colorAccent">#D81B60</color>
</resources>
@@ -0,0 +1,4 @@
<resources>
<string name="app_name" translatable="false">Object Detection 3D</string>
<string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
</resources>
@@ -0,0 +1,11 @@
<resources>
<!-- Base application theme. -->
<style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
<!-- Customize your theme here. -->
<item name="colorPrimary">@color/colorPrimary</item>
<item name="colorPrimaryDark">@color/colorPrimaryDark</item>
<item name="colorAccent">@color/colorAccent</item>
</style>
</resources>
+1 -1
View File
@@ -63,7 +63,7 @@ COPY . /mediapipe/
# Install bazel
ARG BAZEL_VERSION=0.29.1
ARG BAZEL_VERSION=1.1.0
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
+8 -4
View File
@@ -1,9 +1,11 @@
# Coral Dev Board Setup (experimental)
**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may
**Disclaimer**: Running MediaPipe on Coral is experimental, and this process may
not be exact and is subject to change. These instructions have only been tested
on the [Coral Dev Board](https://coral.ai/products/dev-board/) with Mendel 4.0,
and may vary for different devices and workstations.
on the [Coral Dev Board](https://coral.ai/products/dev-board/)
running [Mendel Enterprise Day 13](https://coral.ai/software/) OS and
using [Diploria2](https://github.com/google-coral/edgetpu/tree/diploria2)
edgetpu libs, and may vary for different devices and workstations.
This file describes how to prepare a Coral Dev Board and setup a Linux
Docker container for building MediaPipe applications that run on Edge TPU.
@@ -16,10 +18,12 @@ Docker container for building MediaPipe applications that run on Edge TPU.
* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works
Note: alias mdt="python3 -m mdt.main" may be needed on some systems
* (on coral device) prepare MediaPipe
cd ~
sudo apt-get install -y git
sudo apt-get update && sudo apt-get install -y git
git clone https://github.com/google/mediapipe.git
mkdir mediapipe/bazel-bin
+62 -19
View File
@@ -10,19 +10,25 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1")
versions.check(minimum_bazel_version = "1.0.0",
maximum_bazel_version = "1.2.1")
# ABSL cpp library.
# ABSL cpp library lts_2020_02_25
http_archive(
name = "com_google_absl",
# Head commit on 2019-04-12.
# TODO: Switch to the latest absl version when the problem gets
# fixed.
urls = [
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
],
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [
"@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "abseil-cpp-20200225",
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
)
http_archive(
@@ -72,6 +78,14 @@ http_archive(
],
)
# easyexif
http_archive(
name = "easyexif",
url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
strip_prefix = "easyexif-master",
build_file = "@//third_party:easyexif.BUILD",
)
# libyuv
http_archive(
name = "libyuv",
@@ -103,15 +117,23 @@ http_archive(
],
)
# 2019-11-12
_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
# 2020-02-12
# The last commit before TensorFlow switched to Bazel 2.0
_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
http_archive(
name = "org_tensorflow",
urls = [
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
],
# A compatibility patch
patches = [
"@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
sha256 = _TENSORFLOW_SHA256,
)
@@ -119,8 +141,22 @@ http_archive(
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
tf_workspace(tf_repo_name = "org_tensorflow")
http_archive(
name = "ceres_solver",
url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
patches = [
"@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
],
patch_args = [
"-p1",
],
strip_prefix = "ceres-solver-1.14.0",
sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
)
# Please run
# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
# libopencv-calib3d-dev libopencv-features2d-dev \
# libopencv-imgproc-dev libopencv-video-dev
new_local_repository(
name = "linux_opencv",
@@ -149,11 +185,10 @@ new_local_repository(
http_archive(
name = "android_opencv",
sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
build_file = "@//third_party:opencv_android.BUILD",
strip_prefix = "OpenCV-android-sdk",
type = "zip",
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
)
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
@@ -184,13 +219,18 @@ maven_install(
artifacts = [
"androidx.annotation:annotation:aar:1.1.0",
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
"androidx.camera:camera-core:aar:1.0.0-alpha06",
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
"androidx.core:core:aar:1.1.0-rc03",
"androidx.legacy:legacy-support-v4:aar:1.0.0",
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
"com.google.android.material:material:aar:1.0.0-rc01",
],
repositories = ["https://dl.google.com/dl/android/maven2"],
repositories = [
"https://dl.google.com/dl/android/maven2",
"https://repo1.maven.org/maven2",
],
)
maven_server(
@@ -206,10 +246,10 @@ maven_jar(
)
maven_jar(
name = "androidx_concurrent_futures",
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
server = "google_server",
name = "androidx_concurrent_futures",
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
server = "google_server",
)
maven_jar(
@@ -285,10 +325,13 @@ http_archive(
build_file = "@//third_party:google_toolbox_for_mac.BUILD",
)
### Coral ###
# Coral
#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
# Oct 2019
#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229
#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb
http_archive(
name = "coral_crosstool",
sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
@@ -182,8 +182,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
@@ -173,7 +173,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
+1 -1
View File
@@ -8,7 +8,7 @@ echo ' sh mediapipe/examples/coral/setup.sh '
sleep 3
mkdir opencv32_arm64_libs
mkdir -p opencv32_arm64_libs
cp mediapipe/examples/coral/update_sources.sh update_sources.sh
chmod +x update_sources.sh
+1 -1
View File
@@ -53,7 +53,7 @@ cc_library(
# Linux only.
# Must have a GPU with EGL support:
# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
# (or similar nvidia/amd equivalent)
cc_library(
name = "demo_run_graph_main_gpu",
@@ -11,6 +11,8 @@
2. Build and run the run_autoflip binary to process a local video.
Note: AutoFlip currently only works with OpenCV 3 . Please verify your OpenCV version beforehand.
```bash
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
mediapipe/examples/desktop/autoflip:run_autoflip
@@ -63,12 +63,15 @@ import random
import subprocess
import sys
import tempfile
import urllib
import zipfile
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
from six.moves import range
from six.moves import urllib
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms
@@ -218,7 +221,7 @@ class Charades(object):
return output_dict
if split not in SPLITS:
raise ValueError("Split %s not in %s" % split, str(SPLITS.keys()))
raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys())))
all_shards = tf.io.gfile.glob(
os.path.join(self.path_to_data, SPLITS[split][0] + "-*-of-*"))
random.shuffle(all_shards)
@@ -329,7 +332,7 @@ class Charades(object):
if sys.version_info >= (3, 0):
urlretrieve = urllib.request.urlretrieve
else:
urlretrieve = urllib.urlretrieve
urlretrieve = urllib.request.urlretrieve
logging.info("Creating data directory.")
tf.io.gfile.makedirs(self.path_to_data)
logging.info("Downloading license.")
@@ -57,12 +57,13 @@ import random
import subprocess
import sys
import tempfile
import urllib
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
from six.moves import range
from six.moves import urllib
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms
@@ -198,7 +199,7 @@ class DemoDataset(object):
if sys.version_info >= (3, 0):
urlretrieve = urllib.request.urlretrieve
else:
urlretrieve = urllib.urlretrieve
urlretrieve = urllib.request.urlretrieve
for split in SPLITS:
reader = csv.DictReader(SPLITS[split].split("\n"))
all_metadata = []
@@ -73,12 +73,14 @@ import subprocess
import sys
import tarfile
import tempfile
import urllib
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
from six.moves import range
from six.moves import urllib
from six.moves import zip
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms
@@ -96,15 +98,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
SPLITS = {
"train": {
"shards": 1000,
"examples": 540247
"examples": 538779
},
"validate": {
"shards": 100,
"examples": 34610
"examples": 34499
},
"test": {
"shards": 100,
"examples": 69103
"examples": 68847
},
"custom": {
"csv": None, # Add a CSV for your own data here.
@@ -198,7 +200,7 @@ class Kinetics(object):
return output_dict
if split not in SPLITS:
raise ValueError("Split %s not in %s" % split, str(SPLITS.keys()))
raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys())))
all_shards = tf.io.gfile.glob(
os.path.join(self.path_to_data, FILEPATTERN % split + "-*-of-*"))
random.shuffle(all_shards)
@@ -302,11 +304,12 @@ class Kinetics(object):
continue
# rename the row with a constitent set of names.
if len(csv_row) == 5:
row = dict(zip(["label_name", "video", "start", "end", "split"],
csv_row))
row = dict(
list(
zip(["label_name", "video", "start", "end", "split"],
csv_row)))
else:
row = dict(zip(["video", "start", "end", "split"],
csv_row))
row = dict(list(zip(["video", "start", "end", "split"], csv_row)))
metadata = tf.train.SequenceExample()
ms.set_example_id(bytes23(row["video"] + "_" + row["start"]),
metadata)
@@ -328,7 +331,7 @@ class Kinetics(object):
if sys.version_info >= (3, 0):
urlretrieve = urllib.request.urlretrieve
else:
urlretrieve = urllib.urlretrieve
urlretrieve = urllib.request.urlretrieve
logging.info("Creating data directory.")
tf.io.gfile.makedirs(self.path_to_data)
logging.info("Downloading annotations.")
@@ -404,7 +407,7 @@ class Kinetics(object):
assert NUM_CLASSES == num_keys, (
"Found %d labels for split: %s, should be %d" % (
num_keys, name, NUM_CLASSES))
label_map = dict(zip(classes, range(len(classes))))
label_map = dict(list(zip(classes, list(range(len(classes))))))
if SPLITS[name]["examples"] > 0:
assert SPLITS[name]["examples"] == num_examples, (
"Found %d examples for split: %s, should be %d" % (
@@ -30,6 +30,8 @@
```bash
# cd to the root directory of the MediaPipe repo
cd -
pip3 install tf_slim
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
```
@@ -47,7 +49,7 @@
5. Run the MediaPipe binary to extract the features.
```bash
bazel build -c opt \
bazel build -c opt --linkopt=-s \
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
@@ -87,7 +89,7 @@
3. Build and run the inference binary.
```bash
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
mediapipe/examples/desktop/youtube8m:model_inference
GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \
@@ -113,13 +115,13 @@
2. Build the inference binary.
```bash
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
mediapipe/examples/desktop/youtube8m:model_inference
```
3. Run the python web server.
Note: pip install absl-py
Note: pip3 install absl-py
```bash
python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd`
@@ -142,7 +144,7 @@
3. Build and run the inference binary.
```bash
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' \
bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \
mediapipe/examples/desktop/youtube8m:model_inference
# segment_size is the number of seconds window of frames.
@@ -21,6 +21,7 @@ import sys
from absl import app
from absl import flags
import six
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms
@@ -54,7 +55,7 @@ def main(argv):
ms.set_clip_end_timestamp(
flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata)
with open('/tmp/mediapipe/metadata.pb', 'wb') as writer:
writer.write(metadata.SerializeToString())
writer.write(six.ensure_binary(metadata.SerializeToString()))
if __name__ == '__main__':
@@ -25,7 +25,7 @@ import sys
from absl import app
import tensorflow.compat.v1 as tf
from tensorflow.compat.v1.python.tools import freeze_graph
from tensorflow.python.tools import freeze_graph
BASE_DIR = '/tmp/mediapipe/'
+7 -1
View File
@@ -1078,10 +1078,16 @@ cc_library(
cc_library(
name = "port",
hdrs = ["port.h"],
defines = select({
"//conditions:default": [],
}) + select({
"//conditions:default": [],
"//mediapipe/gpu:disable_gpu": ["MEDIAPIPE_DISABLE_GPU"],
}),
visibility = [
"//mediapipe/framework:__subpackages__",
"//mediapipe/framework/port:__pkg__",
"//mediapipe/util:__pkg__",
"//mediapipe/util:__subpackages__",
],
)
+1 -1
View File
@@ -1301,7 +1301,7 @@ void PrintTimingToInfo(const std::string& label, int64 timer_value) {
"%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: "
"%lld.%06lld)",
days, hours, minutes, seconds, milliseconds, total_seconds,
timer_value % 1000000ll);
timer_value % int64{1000000});
}
bool MetricElementComparator(const std::pair<std::string, int64>& e1,
+1
View File
@@ -251,6 +251,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)
+24
View File
@@ -105,6 +105,30 @@ namespace file {
return ::mediapipe::OkStatus();
}
::mediapipe::Status MatchFileTypeInDirectory(
const std::string& directory, const std::string& file_suffix,
std::vector<std::string>* results) {
DIR* dir = opendir(directory.c_str());
CHECK(dir);
// Iterates through the direcotry.
while (true) {
struct dirent* dir_ent = readdir(dir);
if (dir_ent == nullptr) {
break;
}
if (std::string(dir_ent->d_name) == "." ||
std::string(dir_ent->d_name) == "..") {
continue;
}
if (absl::EndsWith(std::string(dir_ent->d_name), file_suffix)) {
results->push_back(JoinPath(directory, std::string(dir_ent->d_name)));
}
}
closedir(dir);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Exists(absl::string_view file_name) {
struct stat buffer;
int status;
+4
View File
@@ -30,6 +30,10 @@ namespace file {
const std::string& parent_directory, const std::string& file_name,
std::vector<std::string>* results);
::mediapipe::Status MatchFileTypeInDirectory(const std::string& directory,
const std::string& file_suffix,
std::vector<std::string>* results);
::mediapipe::Status Exists(absl::string_view file_name);
} // namespace file
-97
View File
@@ -18,103 +18,6 @@
namespace mediapipe {
Status::Status(::mediapipe::StatusCode code, absl::string_view msg) {
state_ = std::unique_ptr<State>(new State);
state_->code = code;
state_->msg = std::string(msg);
}
void Status::Update(const Status& new_status) {
if (ok()) {
*this = new_status;
}
}
void Status::SlowCopyFrom(const State* src) {
if (src == nullptr) {
state_ = nullptr;
} else {
state_ = std::unique_ptr<State>(new State(*src));
}
}
const std::string& Status::empty_string() {
static std::string* empty = new std::string;
return *empty;
}
std::string Status::ToString() const {
if (state_ == nullptr) {
return "OK";
} else {
char tmp[30];
const char* type;
switch (code()) {
case ::mediapipe::StatusCode::kCancelled:
type = "Cancelled";
break;
case ::mediapipe::StatusCode::kUnknown:
type = "Unknown";
break;
case ::mediapipe::StatusCode::kInvalidArgument:
type = "Invalid argument";
break;
case ::mediapipe::StatusCode::kDeadlineExceeded:
type = "Deadline exceeded";
break;
case ::mediapipe::StatusCode::kNotFound:
type = "Not found";
break;
case ::mediapipe::StatusCode::kAlreadyExists:
type = "Already exists";
break;
case ::mediapipe::StatusCode::kPermissionDenied:
type = "Permission denied";
break;
case ::mediapipe::StatusCode::kUnauthenticated:
type = "Unauthenticated";
break;
case ::mediapipe::StatusCode::kResourceExhausted:
type = "Resource exhausted";
break;
case ::mediapipe::StatusCode::kFailedPrecondition:
type = "Failed precondition";
break;
case ::mediapipe::StatusCode::kAborted:
type = "Aborted";
break;
case ::mediapipe::StatusCode::kOutOfRange:
type = "Out of range";
break;
case ::mediapipe::StatusCode::kUnimplemented:
type = "Unimplemented";
break;
case ::mediapipe::StatusCode::kInternal:
type = "Internal";
break;
case ::mediapipe::StatusCode::kUnavailable:
type = "Unavailable";
break;
case ::mediapipe::StatusCode::kDataLoss:
type = "Data loss";
break;
default:
snprintf(tmp, sizeof(tmp), "Unknown code(%d)",
static_cast<int>(code()));
type = tmp;
break;
}
std::string result(type);
result += ": ";
result += state_->msg;
return result;
}
}
void Status::IgnoreError() const {
// no-op
}
std::ostream& operator<<(std::ostream& os, const Status& x) {
os << x.ToString();
return os;
+4 -114
View File
@@ -20,126 +20,16 @@
#include <memory>
#include <string>
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
enum class StatusCode {
kOk = 0,
kCancelled = 1,
kUnknown = 2,
kInvalidArgument = 3,
kDeadlineExceeded = 4,
kNotFound = 5,
kAlreadyExists = 6,
kPermissionDenied = 7,
kResourceExhausted = 8,
kFailedPrecondition = 9,
kAborted = 10,
kOutOfRange = 11,
kUnimplemented = 12,
kInternal = 13,
kUnavailable = 14,
kDataLoss = 15,
kUnauthenticated = 16,
kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20
};
using Status = absl::Status;
using StatusCode = absl::StatusCode;
#if defined(__clang__)
// Only clang supports warn_unused_result as a type annotation.
class ABSL_MUST_USE_RESULT Status;
#endif
// Denotes success or failure of a call in MediaPipe.
class Status {
public:
// Creates a success status.
Status() {}
// Creates a status with the specified error code and msg as a
// human-readable std::string containing more detailed information.
Status(::mediapipe::StatusCode code, absl::string_view msg);
// Copies the specified status.
Status(const Status& s);
void operator=(const Status& s);
// Returns true iff the status indicates success.
bool ok() const {
return (state_ == NULL) || (state_->code == ::mediapipe::StatusCode::kOk);
}
::mediapipe::StatusCode code() const {
return ok() ? ::mediapipe::StatusCode::kOk : state_->code;
}
const std::string& error_message() const {
return ok() ? empty_string() : state_->msg;
}
absl::string_view message() const {
return absl::string_view(error_message());
}
bool operator==(const Status& x) const;
bool operator!=(const Status& x) const;
// If `ok()`, stores `new_status` into `*this`. If `!ok()`,
// preserves the current status, but may augment with additional
// information about `new_status`.
//
// Convenient way of keeping track of the first error encountered.
// Instead of:
// `if (overall_status.ok()) overall_status = new_status`
// Use:
// `overall_status.Update(new_status);`
void Update(const Status& new_status);
// Returns a std::string representation of this status suitable for
// printing. Returns the std::string `"OK"` for success.
std::string ToString() const;
// Ignores any errors. This method does nothing except potentially suppress
// complaints from any tools that are checking that errors are not dropped on
// the floor.
void IgnoreError() const;
private:
static const std::string& empty_string();
struct State {
::mediapipe::StatusCode code;
std::string msg;
};
// OK status has a `NULL` state_. Otherwise, `state_` points to
// a `State` structure containing the error code and message(s)
std::unique_ptr<State> state_;
void SlowCopyFrom(const State* src);
};
inline Status::Status(const Status& s)
: state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}
inline void Status::operator=(const Status& s) {
// The following condition catches both aliasing (when this == &s),
// and the common case where both s and *this are ok.
if (state_ != s.state_) {
SlowCopyFrom(s.state_.get());
}
}
inline bool Status::operator==(const Status& x) const {
return (this->state_ == x.state_) || (ToString() == x.ToString());
}
inline bool Status::operator!=(const Status& x) const { return !(*this == x); }
inline Status OkStatus() { return Status(); }
std::ostream& operator<<(std::ostream& os, const Status& x);
typedef std::function<void(const Status&)> StatusCallback;
inline ::mediapipe::Status OkStatus() { return absl::OkStatus(); }
extern std::string* MediaPipeCheckOpHelperOutOfLine(
const ::mediapipe::Status& v, const char* msg);
+3 -3
View File
@@ -72,12 +72,12 @@ StatusBuilder::operator Status() && {
std::string message;
if (join_style_ == MessageJoinStyle::kAnnotate) {
if (!status_.ok()) {
message = absl::StrCat(status_.error_message(), "; ", stream_->str());
message = absl::StrCat(status_.message(), "; ", stream_->str());
}
} else {
message = join_style_ == MessageJoinStyle::kPrepend
? absl::StrCat(stream_->str(), status_.error_message())
: absl::StrCat(status_.error_message(), stream_->str());
? absl::StrCat(stream_->str(), status_.message())
: absl::StrCat(status_.message(), stream_->str());
}
return Status(status_.code(), message);
}
@@ -27,7 +27,7 @@ TEST(StatusBuilder, AnnotateMode) {
<< "annotated message2";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kNotFound);
EXPECT_EQ(status.error_message(),
EXPECT_EQ(status.message(),
"original message; annotated message1 annotated message2");
}
@@ -42,7 +42,7 @@ TEST(StatusBuilder, PrependMode) {
<< "prepended message2 ";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
EXPECT_EQ(status.error_message(),
EXPECT_EQ(status.message(),
"prepended message1 prepended message2 original message");
}
@@ -56,8 +56,7 @@ TEST(StatusBuilder, AppendMode) {
<< " extra message2";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInternal);
EXPECT_EQ(status.error_message(),
"original message extra message1 extra message2");
EXPECT_EQ(status.message(), "original message extra message1 extra message2");
}
TEST(StatusBuilder, NoLoggingMode) {
@@ -69,7 +68,7 @@ TEST(StatusBuilder, NoLoggingMode) {
<< " extra message";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnavailable);
EXPECT_EQ(status.error_message(), "original message");
EXPECT_EQ(status.message(), "original message");
}
} // namespace mediapipe

Some files were not shown because too many files have changed in this diff Show More