diff --git a/.bazelrc b/.bazelrc index 7817ca08..4ffb73bd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,7 +3,7 @@ # Basic build settings build --jobs 128 build --define='absl=1' -build --cxxopt='-std=c++11' +build --cxxopt='-std=c++14' build --copt='-Wno-sign-compare' build --copt='-Wno-unused-function' build --copt='-Wno-uninitialized' diff --git a/README.md b/README.md index 56cd7bac..e7a9c10d 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,10 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172) ## Events -* [ML Conference, Berlin 9-11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/) +* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/) +* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038) +* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/) +* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/) * [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019 * [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California * [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan diff --git a/WORKSPACE b/WORKSPACE index f568e8e9..ca87c83c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -10,8 +10,7 @@ http_archive( sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", ) load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "0.24.1", - maximum_bazel_version = "0.29.1") +versions.check(minimum_bazel_version = "0.24.1") # ABSL cpp library. http_archive( @@ -104,9 +103,9 @@ http_archive( ], ) -# 2019-08-15 -_TENSORFLOW_GIT_COMMIT = "67def62936e28f97c16182dfcc467d8d1cae02b4" -_TENSORFLOW_SHA256= "ddd4e3c056e7c0ff2ef29133b30fa62781dfbf8a903e99efb91a02d292fa9562" +# 2019-11-12 +_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82" +_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075" http_archive( name = "org_tensorflow", urls = [ @@ -115,13 +114,6 @@ http_archive( ], strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, sha256 = _TENSORFLOW_SHA256, - patches = [ - "@//third_party:tensorflow_065c20bf79253257c87bd4614bb9a7fdef015cbb.diff", - "@//third_party:tensorflow_f67fcbefce906cd419e4657f0d41e21019b71abd.diff", - ], - patch_args = [ - "-p1", - ], ) load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace") @@ -255,18 +247,11 @@ android_sdk_repository( # iOS basic build deps. -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -git_repository( +http_archive( name = "build_bazel_rules_apple", - remote = "https://github.com/bazelbuild/rules_apple.git", - tag = "0.18.0", - patches = [ - "@//third_party:rules_apple_c0863d0596ae6b769a29fa3fb72ff036444fd249.diff", - ], - patch_args = [ - "-p1", - ], + sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", + strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", + url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", ) load( diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 6324cce1..86cb2852 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -13,12 +13,12 @@ # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:private"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "concatenate_vector_calculator_proto", srcs = ["concatenate_vector_calculator.proto"], @@ -79,6 +79,13 @@ proto_library( ], ) +proto_library( + name = "clip_vector_size_calculator_proto", + srcs = ["clip_vector_size_calculator.proto"], + visibility = ["//visibility:public"], + deps = ["//mediapipe/framework:calculator_proto"], +) + mediapipe_cc_proto_library( name = "packet_cloner_calculator_cc_proto", srcs = ["packet_cloner_calculator.proto"], @@ -111,6 +118,14 @@ mediapipe_cc_proto_library( deps = [":concatenate_vector_calculator_proto"], ) +mediapipe_cc_proto_library( + name = "clip_vector_size_calculator_cc_proto", + srcs = ["clip_vector_size_calculator.proto"], + cc_deps = ["//mediapipe/framework:calculator_cc_proto"], + visibility = ["//visibility:public"], + deps = [":clip_vector_size_calculator_proto"], +) + mediapipe_cc_proto_library( name = "dequantize_byte_array_calculator_cc_proto", srcs = ["dequantize_byte_array_calculator.proto"], @@ -169,6 +184,66 @@ cc_test( ], ) +cc_library( + name = "begin_loop_calculator", + srcs = ["begin_loop_calculator.cc"], + hdrs = ["begin_loop_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_contract", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/memory", + ], + alwayslink = 1, +) + +cc_library( + name = "end_loop_calculator", + srcs = ["end_loop_calculator.cc"], + hdrs = ["end_loop_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_contract", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/util:render_data_cc_proto", + ], + alwayslink = 1, +) + +cc_test( + name = "begin_end_loop_calculator_graph_test", + srcs = ["begin_end_loop_calculator_graph_test.cc"], + deps = [ + ":begin_loop_calculator", + ":end_loop_calculator", + "//mediapipe/calculators/core:packet_cloner_calculator", + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_contract", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/memory", + ], +) + cc_library( name = "concatenate_vector_calculator", srcs = ["concatenate_vector_calculator.cc"], @@ -219,6 +294,50 @@ cc_test( ], ) +cc_library( + name = "clip_vector_size_calculator", + srcs = ["clip_vector_size_calculator.cc"], + hdrs = ["clip_vector_size_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + ":clip_vector_size_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@org_tensorflow//tensorflow/lite:framework", + ], + alwayslink = 1, +) + +cc_library( + name = "clip_detection_vector_size_calculator", + srcs = ["clip_detection_vector_size_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":clip_vector_size_calculator", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:detection_cc_proto", + ], + alwayslink = 1, +) + +cc_test( + name = "clip_vector_size_calculator_test", + srcs = ["clip_vector_size_calculator_test.cc"], + deps = [ + ":clip_vector_size_calculator", + "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "counting_source_calculator", srcs = ["counting_source_calculator.cc"], @@ -300,7 +419,7 @@ cc_library( "//visibility:public", ], deps = [ - "//mediapipe/calculators/core:packet_cloner_calculator_cc_proto", + ":packet_cloner_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "@com_google_absl//absl/strings", ], diff --git a/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc b/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc new file mode 100644 index 00000000..03e1a443 --- /dev/null +++ b/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc @@ -0,0 +1,335 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "absl/memory/memory.h" +#include "mediapipe/calculators/core/begin_loop_calculator.h" +#include "mediapipe/calculators/core/end_loop_calculator.h" +#include "mediapipe/framework/calculator_contract.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" // NOLINT + +namespace mediapipe { +namespace { + +typedef BeginLoopCalculator> BeginLoopIntegerCalculator; +REGISTER_CALCULATOR(BeginLoopIntegerCalculator); + +class IncrementCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Index(0).Set(); + cc->Outputs().Index(0).Set(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + const int& input_int = cc->Inputs().Index(0).Get(); + auto output_int = absl::make_unique(input_int + 1); + cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } +}; + +REGISTER_CALCULATOR(IncrementCalculator); + +typedef EndLoopCalculator> EndLoopIntegersCalculator; +REGISTER_CALCULATOR(EndLoopIntegersCalculator); + +class BeginEndLoopCalculatorGraphTest : public ::testing::Test { + protected: + BeginEndLoopCalculatorGraphTest() { + graph_config_ = ParseTextProtoOrDie( + R"( + num_threads: 4 + input_stream: "ints" + node { + calculator: "BeginLoopIntegerCalculator" + input_stream: "ITERABLE:ints" + output_stream: "ITEM:int" + output_stream: "BATCH_END:timestamp" + } + node { + calculator: "IncrementCalculator" + input_stream: "int" + output_stream: "int_plus_one" + } + node { + calculator: "EndLoopIntegersCalculator" + input_stream: "ITEM:int_plus_one" + input_stream: "BATCH_END:timestamp" + output_stream: "ITERABLE:ints_plus_one" + } + )"); + tool::AddVectorSink("ints_plus_one", &graph_config_, &output_packets_); + } + + CalculatorGraphConfig graph_config_; + std::vector output_packets_; +}; + +TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + auto input_vector = absl::make_unique>(); + Timestamp input_timestamp = Timestamp(0); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector.release()).At(input_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + // EndLoopCalc will forward the timestamp bound because there are no elements + // in collection to output. + ASSERT_EQ(0, output_packets_.size()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + auto input_vector = absl::make_unique>(); + input_vector->emplace_back(0); + input_vector->emplace_back(1); + input_vector->emplace_back(2); + Timestamp input_timestamp = Timestamp(0); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector.release()).At(input_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + ASSERT_EQ(1, output_packets_.size()); + EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp()); + std::vector expected_output_vector = {1, 2, 3}; + EXPECT_EQ(expected_output_vector, output_packets_[0].Get>()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + + auto input_vector0 = absl::make_unique>(); + input_vector0->emplace_back(0); + input_vector0->emplace_back(1); + Timestamp input_timestamp0 = Timestamp(0); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector0.release()).At(input_timestamp0))); + + auto input_vector1 = absl::make_unique>(); + Timestamp input_timestamp1 = Timestamp(1); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector1.release()).At(input_timestamp1))); + + auto input_vector2 = absl::make_unique>(); + input_vector2->emplace_back(2); + input_vector2->emplace_back(3); + Timestamp input_timestamp2 = Timestamp(2); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector2.release()).At(input_timestamp2))); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + + ASSERT_EQ(2, output_packets_.size()); + + EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp()); + std::vector expected_output_vector0 = {1, 2}; + EXPECT_EQ(expected_output_vector0, + output_packets_[0].Get>()); + + // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are + // no elements in vector to process. + + EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp()); + std::vector expected_output_vector2 = {3, 4}; + EXPECT_EQ(expected_output_vector2, + output_packets_[1].Get>()); +} + +class MultiplierCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Index(0).Set(); + cc->Inputs().Index(1).Set(); + cc->Outputs().Index(0).Set(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + const int& input_int = cc->Inputs().Index(0).Get(); + const int& multiplier_int = cc->Inputs().Index(1).Get(); + auto output_int = absl::make_unique(input_int * multiplier_int); + cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } +}; + +REGISTER_CALCULATOR(MultiplierCalculator); + +class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test { + protected: + BeginEndLoopCalculatorGraphWithClonedInputsTest() { + graph_config_ = ParseTextProtoOrDie( + R"( + num_threads: 4 + input_stream: "ints" + input_stream: "multiplier" + node { + calculator: "BeginLoopIntegerCalculator" + input_stream: "ITERABLE:ints" + input_stream: "CLONE:multiplier" + output_stream: "ITEM:int_at_loop" + output_stream: "CLONE:multiplier_cloned_at_loop" + output_stream: "BATCH_END:timestamp" + } + node { + calculator: "MultiplierCalculator" + input_stream: "int_at_loop" + input_stream: "multiplier_cloned_at_loop" + output_stream: "multiplied_int_at_loop" + } + node { + calculator: "EndLoopIntegersCalculator" + input_stream: "ITEM:multiplied_int_at_loop" + input_stream: "BATCH_END:timestamp" + output_stream: "ITERABLE:multiplied_ints" + } + )"); + tool::AddVectorSink("multiplied_ints", &graph_config_, &output_packets_); + } + + CalculatorGraphConfig graph_config_; + std::vector output_packets_; +}; + +TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + auto input_vector = absl::make_unique>(); + Timestamp input_timestamp = Timestamp(42); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector.release()).At(input_timestamp))); + auto multiplier = absl::make_unique(2); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "multiplier", Adopt(multiplier.release()).At(input_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + // EndLoopCalc will forward the timestamp bound because there are no elements + // in collection to output. + ASSERT_EQ(0, output_packets_.size()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + auto input_vector = absl::make_unique>(); + input_vector->emplace_back(0); + input_vector->emplace_back(1); + input_vector->emplace_back(2); + Timestamp input_timestamp = Timestamp(42); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector.release()).At(input_timestamp))); + auto multiplier = absl::make_unique(2); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "multiplier", Adopt(multiplier.release()).At(input_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + ASSERT_EQ(1, output_packets_.size()); + EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp()); + std::vector expected_output_vector = {0, 2, 4}; + EXPECT_EQ(expected_output_vector, output_packets_[0].Get>()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) { + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config_)); + MP_EXPECT_OK(graph.StartRun({})); + + auto input_vector0 = absl::make_unique>(); + input_vector0->emplace_back(0); + input_vector0->emplace_back(1); + Timestamp input_timestamp0 = Timestamp(42); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector0.release()).At(input_timestamp0))); + auto multiplier0 = absl::make_unique(2); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "multiplier", Adopt(multiplier0.release()).At(input_timestamp0))); + + auto input_vector1 = absl::make_unique>(); + Timestamp input_timestamp1 = Timestamp(43); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector1.release()).At(input_timestamp1))); + auto multiplier1 = absl::make_unique(2); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "multiplier", Adopt(multiplier1.release()).At(input_timestamp1))); + + auto input_vector2 = absl::make_unique>(); + input_vector2->emplace_back(2); + input_vector2->emplace_back(3); + Timestamp input_timestamp2 = Timestamp(44); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "ints", Adopt(input_vector2.release()).At(input_timestamp2))); + auto multiplier2 = absl::make_unique(3); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "multiplier", Adopt(multiplier2.release()).At(input_timestamp2))); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + + ASSERT_EQ(2, output_packets_.size()); + + EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp()); + std::vector expected_output_vector0 = {0, 2}; + EXPECT_EQ(expected_output_vector0, + output_packets_[0].Get>()); + + // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are + // no elements in vector to process. + + EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp()); + std::vector expected_output_vector2 = {6, 9}; + EXPECT_EQ(expected_output_vector2, + output_packets_[1].Get>()); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc new file mode 100644 index 00000000..cc7f6b85 --- /dev/null +++ b/mediapipe/calculators/core/begin_loop_calculator.cc @@ -0,0 +1,40 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/core/begin_loop_calculator.h" + +#include + +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe { + +// A calculator to process std::vector. +typedef BeginLoopCalculator> + BeginLoopNormalizedLandmarkCalculator; +REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator); + +// A calculator to process std::vector>. +typedef BeginLoopCalculator< + std::vector>> + BeginLoopNormalizedLandmarksVectorCalculator; +REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator); + +// A calculator to process std::vector. +typedef BeginLoopCalculator> + BeginLoopNormalizedRectCalculator; +REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/begin_loop_calculator.h b/mediapipe/calculators/core/begin_loop_calculator.h new file mode 100644 index 00000000..7258b4bf --- /dev/null +++ b/mediapipe/calculators/core/begin_loop_calculator.h @@ -0,0 +1,157 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ + +#include "absl/memory/memory.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_contract.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// Calculator for implementing loops on iterable collections inside a MediaPipe +// graph. +// +// It is designed to be used like: +// +// node { +// calculator: "BeginLoopWithIterableCalculator" +// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts +// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts +// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts +// } +// +// node { +// calculator: "ElementToBlaConverterSubgraph" +// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts +// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts +// } +// +// node { +// calculator: "EndLoopWithOutputCalculator" +// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts +// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts +// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts +// } +// +// BeginLoopCalculator accepts an optional input stream tagged with "TICK" +// which if non-empty, wakes up the calculator and calls +// BeginLoopCalculator::Process(). Input streams tagged with "CLONE" are cloned +// to the corresponding output streams at loop timestamps. This ensures that a +// MediaPipe graph or sub-graph can run multiple times, once per element in the +// "ITERABLE" for each pakcet clone of the packets in the "CLONE" input streams. +template +class BeginLoopCalculator : public CalculatorBase { + using ItemT = typename IterableT::value_type; + + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + // A non-empty packet in the optional "TICK" input stream wakes up the + // calculator. + if (cc->Inputs().HasTag("TICK")) { + cc->Inputs().Tag("TICK").SetAny(); + } + + // An iterable collection in the input stream. + RET_CHECK(cc->Inputs().HasTag("ITERABLE")); + cc->Inputs().Tag("ITERABLE").Set(); + + // An element from the collection. + RET_CHECK(cc->Outputs().HasTag("ITEM")); + cc->Outputs().Tag("ITEM").Set(); + + RET_CHECK(cc->Outputs().HasTag("BATCH_END")); + cc->Outputs() + .Tag("BATCH_END") + .Set( + // A flush signal to the corresponding EndLoopCalculator for it to + // emit the aggregated result with the timestamp contained in this + // flush signal packet. + ); + + // Input streams tagged with "CLONE" are cloned to the corresponding + // "CLONE" output streams at loop timestamps. + RET_CHECK(cc->Inputs().NumEntries("CLONE") == + cc->Outputs().NumEntries("CLONE")); + if (cc->Inputs().NumEntries("CLONE") > 0) { + for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) { + cc->Inputs().Get("CLONE", i).SetAny(); + cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i)); + } + } + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) final { + Timestamp last_timestamp = loop_internal_timestamp_; + if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) { + const IterableT& collection = + cc->Inputs().Tag("ITERABLE").template Get(); + for (const auto& item : collection) { + cc->Outputs().Tag("ITEM").AddPacket( + MakePacket(item).At(loop_internal_timestamp_)); + ForwardClonePackets(cc, loop_internal_timestamp_); + ++loop_internal_timestamp_; + } + } + + // The collection was empty and nothing was processed. + if (last_timestamp == loop_internal_timestamp_) { + // Increment loop_internal_timestamp_ because it is used up now. + ++loop_internal_timestamp_; + for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) { + it->SetNextTimestampBound(loop_internal_timestamp_); + } + } + + // The for loop processing the input collection already incremented + // loop_internal_timestamp_. To emit BATCH_END packet along the last + // non-BATCH_END packet, decrement by one. + cc->Outputs() + .Tag("BATCH_END") + .AddPacket(MakePacket(cc->InputTimestamp()) + .At(Timestamp(loop_internal_timestamp_ - 1))); + + return ::mediapipe::OkStatus(); + } + + private: + void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) { + if (cc->Inputs().NumEntries("CLONE") > 0) { + for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) { + if (!cc->Inputs().Get("CLONE", i).IsEmpty()) { + auto input_packet = cc->Inputs().Get("CLONE", i).Value(); + cc->Outputs() + .Get("CLONE", i) + .AddPacket(std::move(input_packet).At(output_timestamp)); + } + } + } + } + + // Fake timestamps generated per element in collection. + Timestamp loop_internal_timestamp_ = Timestamp(0); +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc b/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc new file mode 100644 index 00000000..55bcf2fe --- /dev/null +++ b/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc @@ -0,0 +1,26 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "mediapipe/calculators/core/clip_vector_size_calculator.h" +#include "mediapipe/framework/formats/detection.pb.h" + +namespace mediapipe { + +typedef ClipVectorSizeCalculator<::mediapipe::Detection> + ClipDetectionVectorSizeCalculator; +REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.cc b/mediapipe/calculators/core/clip_vector_size_calculator.cc new file mode 100644 index 00000000..388cc3a6 --- /dev/null +++ b/mediapipe/calculators/core/clip_vector_size_calculator.cc @@ -0,0 +1,28 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/core/clip_vector_size_calculator.h" + +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe { + +typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect> + ClipNormalizedRectVectorSizeCalculator; +REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.h b/mediapipe/calculators/core/clip_vector_size_calculator.h new file mode 100644 index 00000000..f8e12ff2 --- /dev/null +++ b/mediapipe/calculators/core/clip_vector_size_calculator.h @@ -0,0 +1,137 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ + +#include +#include + +#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// Clips the size of the input vector of type T to a specified max_vec_size. +// In a graph it will be used as: +// node { +// calculator: "ClipIntVectorSizeCalculator" +// input_stream: "input_vector" +// output_stream: "output_vector" +// options { +// [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] { +// max_vec_size: 5 +// } +// } +// } +template +class ClipVectorSizeCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + RET_CHECK(cc->Inputs().NumEntries() == 1); + RET_CHECK(cc->Outputs().NumEntries() == 1); + + if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>() + .max_vec_size() < 1) { + return ::mediapipe::InternalError( + "max_vec_size should be greater than or equal to 1."); + } + + cc->Inputs().Index(0).Set>(); + cc->Outputs().Index(0).Set>(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>() + .max_vec_size(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + if (max_vec_size_ < 1) { + return ::mediapipe::InternalError( + "max_vec_size should be greater than or equal to 1."); + } + if (cc->Inputs().Index(0).IsEmpty()) { + return ::mediapipe::OkStatus(); + } + + return ClipVectorSize(std::is_copy_constructible(), cc); + } + + template + ::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) { + auto output = absl::make_unique>(); + const std::vector& input_vector = + cc->Inputs().Index(0).Get>(); + if (max_vec_size_ >= input_vector.size()) { + output->insert(output->end(), input_vector.begin(), input_vector.end()); + } else { + for (int i = 0; i < max_vec_size_; ++i) { + output->push_back(input_vector[i]); + } + } + cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } + + template + ::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) { + return ConsumeAndClipVectorSize(std::is_move_constructible(), cc); + } + + template + ::mediapipe::Status ConsumeAndClipVectorSize(std::true_type, + CalculatorContext* cc) { + auto output = absl::make_unique>(); + ::mediapipe::StatusOr>> input_status = + cc->Inputs().Index(0).Value().Consume>(); + + if (input_status.ok()) { + std::unique_ptr> input_vector = + std::move(input_status).ValueOrDie(); + auto begin_it = input_vector->begin(); + auto end_it = input_vector->end(); + if (max_vec_size_ < input_vector->size()) { + end_it = input_vector->begin() + max_vec_size_; + } + output->insert(output->end(), std::make_move_iterator(begin_it), + std::make_move_iterator(end_it)); + } else { + return input_status.status(); + } + cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } + + template + ::mediapipe::Status ConsumeAndClipVectorSize(std::false_type, + CalculatorContext* cc) { + return ::mediapipe::InternalError( + "Cannot copy or move input vectors and clip their size."); + } + + private: + int max_vec_size_ = 0; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.proto b/mediapipe/calculators/core/clip_vector_size_calculator.proto new file mode 100644 index 00000000..5dea660d --- /dev/null +++ b/mediapipe/calculators/core/clip_vector_size_calculator.proto @@ -0,0 +1,28 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message ClipVectorSizeCalculatorOptions { + extend CalculatorOptions { + optional ClipVectorSizeCalculatorOptions ext = 274674998; + } + + // Maximum size of output vector. + optional int32 max_vec_size = 1 [default = 1]; +} diff --git a/mediapipe/calculators/core/clip_vector_size_calculator_test.cc b/mediapipe/calculators/core/clip_vector_size_calculator_test.cc new file mode 100644 index 00000000..60a42120 --- /dev/null +++ b/mediapipe/calculators/core/clip_vector_size_calculator_test.cc @@ -0,0 +1,179 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/core/clip_vector_size_calculator.h" + +#include +#include +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" // NOLINT + +namespace mediapipe { + +typedef ClipVectorSizeCalculator TestClipIntVectorSizeCalculator; +REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator); + +void AddInputVector(const std::vector& input, int64 timestamp, + CalculatorRunner* runner) { + runner->MutableInputs()->Index(0).packets.push_back( + MakePacket>(input).At(Timestamp(timestamp))); +} + +TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) { + CalculatorGraphConfig::Node node_config = + ParseTextProtoOrDie(R"( + calculator: "TestClipIntVectorSizeCalculator" + input_stream: "input_vector" + output_stream: "output_vector" + options { + [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 } + } + )"); + CalculatorRunner runner(node_config); + + std::vector input = {}; + AddInputVector(input, /*timestamp=*/1, &runner); + MP_ASSERT_OK(runner.Run()); + + const std::vector& outputs = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, outputs.size()); + EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); + EXPECT_TRUE(outputs[0].Get>().empty()); +} + +TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) { + CalculatorGraphConfig::Node node_config = + ParseTextProtoOrDie(R"( + calculator: "TestClipIntVectorSizeCalculator" + input_stream: "input_vector" + output_stream: "output_vector" + options { + [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 } + } + )"); + CalculatorRunner runner(node_config); + + std::vector input = {0, 1, 2, 3}; + AddInputVector(input, /*timestamp=*/1, &runner); + MP_ASSERT_OK(runner.Run()); + + const std::vector& outputs = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, outputs.size()); + EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); + const std::vector& output = outputs[0].Get>(); + EXPECT_EQ(2, output.size()); + std::vector expected_vector = {0, 1}; + EXPECT_EQ(expected_vector, output); +} + +TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) { + CalculatorGraphConfig::Node node_config = + ParseTextProtoOrDie(R"( + calculator: "TestClipIntVectorSizeCalculator" + input_stream: "input_vector" + output_stream: "output_vector" + options { + [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 } + } + )"); + CalculatorRunner runner(node_config); + + { + std::vector input = {0, 1, 2, 3}; + AddInputVector(input, /*timestamp=*/1, &runner); + } + { + std::vector input = {2, 3, 4, 5}; + AddInputVector(input, /*timestamp=*/2, &runner); + } + MP_ASSERT_OK(runner.Run()); + + const std::vector& outputs = runner.Outputs().Index(0).packets; + EXPECT_EQ(2, outputs.size()); + { + EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); + const std::vector& output = outputs[0].Get>(); + EXPECT_EQ(3, output.size()); + std::vector expected_vector = {0, 1, 2}; + EXPECT_EQ(expected_vector, output); + } + { + EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); + const std::vector& output = outputs[1].Get>(); + EXPECT_EQ(3, output.size()); + std::vector expected_vector = {2, 3, 4}; + EXPECT_EQ(expected_vector, output); + } +} + +typedef ClipVectorSizeCalculator> + TestClipUniqueIntPtrVectorSizeCalculator; +REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator); + +TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) { + /* Note: We don't use CalculatorRunner for this test because it keeps copies + * of input packets, so packets sent to the graph don't have sole ownership. + * The test needs to send packets that own the data. + */ + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(R"( + input_stream: "input_vector" + node { + calculator: "TestClipUniqueIntPtrVectorSizeCalculator" + input_stream: "input_vector" + output_stream: "output_vector" + options { + [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 } + } + } + )"); + + std::vector outputs; + tool::AddVectorSink("output_vector", &graph_config, &outputs); + + CalculatorGraph graph; + MP_EXPECT_OK(graph.Initialize(graph_config)); + MP_EXPECT_OK(graph.StartRun({})); + + // input1 : {0, 1, 2, 3, 4, 5} + auto input_vector = absl::make_unique>>(6); + for (int i = 0; i < 6; ++i) { + input_vector->at(i) = absl::make_unique(i); + } + + MP_EXPECT_OK(graph.AddPacketToInputStream( + "input_vector", Adopt(input_vector.release()).At(Timestamp(1)))); + + MP_EXPECT_OK(graph.WaitUntilIdle()); + MP_EXPECT_OK(graph.CloseAllPacketSources()); + MP_EXPECT_OK(graph.WaitUntilDone()); + + EXPECT_EQ(1, outputs.size()); + EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); + const std::vector>& result = + outputs[0].Get>>(); + EXPECT_EQ(3, result.size()); + for (int i = 0; i < 3; ++i) { + const std::unique_ptr& v = result[i]; + EXPECT_EQ(i, *v); + } +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc new file mode 100644 index 00000000..8991e97f --- /dev/null +++ b/mediapipe/calculators/core/end_loop_calculator.cc @@ -0,0 +1,45 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/core/end_loop_calculator.h" + +#include + +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe { + +typedef EndLoopCalculator> + EndLoopNormalizedRectCalculator; +REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator); + +typedef EndLoopCalculator> + EndLoopNormalizedLandmarkCalculator; +REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator); + +typedef EndLoopCalculator< + std::vector>> + EndLoopNormalizedLandmarksVectorCalculator; +REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator); + +typedef EndLoopCalculator> EndLoopBooleanCalculator; +REGISTER_CALCULATOR(EndLoopBooleanCalculator); + +typedef EndLoopCalculator> + EndLoopRenderDataCalculator; +REGISTER_CALCULATOR(EndLoopRenderDataCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.h b/mediapipe/calculators/core/end_loop_calculator.h new file mode 100644 index 00000000..869cc29a --- /dev/null +++ b/mediapipe/calculators/core/end_loop_calculator.h @@ -0,0 +1,106 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ + +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_contract.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// Calculator for completing the processing of loops on iterable collections +// inside a MediaPipe graph. The EndLoopCalculator collects all input packets +// from ITEM input_stream into a collection and upon receiving the flush signal +// from the "BATCH_END" tagged input stream, it emits the aggregated results +// at the original timestamp contained in the "BATCH_END" input stream. +// +// It is designed to be used like: +// +// node { +// calculator: "BeginLoopWithIterableCalculator" +// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts +// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts +// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts +// } +// +// node { +// calculator: "ElementToBlaConverterSubgraph" +// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts +// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts +// } +// +// node { +// calculator: "EndLoopWithOutputCalculator" +// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts +// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts +// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts +// } +template +class EndLoopCalculator : public CalculatorBase { + using ItemT = typename IterableT::value_type; + + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + RET_CHECK(cc->Inputs().HasTag("BATCH_END")) + << "Missing BATCH_END tagged input_stream."; + cc->Inputs().Tag("BATCH_END").Set(); + + RET_CHECK(cc->Inputs().HasTag("ITEM")); + cc->Inputs().Tag("ITEM").Set(); + + RET_CHECK(cc->Outputs().HasTag("ITERABLE")); + cc->Outputs().Tag("ITERABLE").Set(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + if (!cc->Inputs().Tag("ITEM").IsEmpty()) { + if (!input_stream_collection_) { + input_stream_collection_.reset(new IterableT); + } + input_stream_collection_->push_back( + cc->Inputs().Tag("ITEM").template Get()); + } + + if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal + Timestamp loop_control_ts = + cc->Inputs().Tag("BATCH_END").template Get(); + if (input_stream_collection_) { + cc->Outputs() + .Tag("ITERABLE") + .Add(input_stream_collection_.release(), loop_control_ts); + } else { + // Since there is no collection, inform downstream calculators to not + // expect any packet by updating the timestamp bounds. + cc->Outputs() + .Tag("ITERABLE") + .SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1)); + } + } + return ::mediapipe::OkStatus(); + } + + private: + std::unique_ptr input_stream_collection_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/packet_resampler_calculator.cc b/mediapipe/calculators/core/packet_resampler_calculator.cc index 4271435f..2966e458 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.cc +++ b/mediapipe/calculators/core/packet_resampler_calculator.cc @@ -74,6 +74,12 @@ class PacketResamplerCalculator : public CalculatorBase { ::mediapipe::Status Process(CalculatorContext* cc) override; private: + // Calculates the first sampled timestamp that incorporates a jittering + // offset. + void InitializeNextOutputTimestampWithJitter(); + // Calculates the next sampled timestamp that incorporates a jittering offset. + void UpdateNextOutputTimestampWithJitter(); + // Logic for Process() when jitter_ != 0.0. ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc); @@ -233,6 +239,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) { << Timestamp::kTimestampUnitsPerSecond; frame_time_usec_ = static_cast(1000000.0 / frame_rate_); + video_header_.frame_rate = frame_rate_; if (resampler_options.output_header() != @@ -295,6 +302,17 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) { return ::mediapipe::OkStatus(); } +void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() { + next_output_timestamp_ = + first_timestamp_ + frame_time_usec_ * random_->RandFloat(); +} + +void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() { + next_output_timestamp_ += + frame_time_usec_ * + ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat()); +} + ::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter( CalculatorContext* cc) { RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream()); @@ -302,8 +320,13 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) { if (first_timestamp_ == Timestamp::Unset()) { first_timestamp_ = cc->InputTimestamp(); - next_output_timestamp_ = - first_timestamp_ + frame_time_usec_ * random_->RandFloat(); + InitializeNextOutputTimestampWithJitter(); + if (first_timestamp_ == next_output_timestamp_) { + OutputWithinLimits( + cc, + cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_)); + UpdateNextOutputTimestampWithJitter(); + } return ::mediapipe::OkStatus(); } @@ -322,9 +345,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) { ? last_packet_ : cc->Inputs().Get(input_data_id_).Value()) .At(next_output_timestamp_)); - next_output_timestamp_ += - frame_time_usec_ * - ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat()); + UpdateNextOutputTimestampWithJitter(); return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index b8fdbdfa..5a0a7561 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:private"]) exports_files(["LICENSE"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "opencv_image_encoder_calculator_proto", srcs = ["opencv_image_encoder_calculator.proto"], diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index c9c50549..8c9d0df8 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -13,12 +13,12 @@ # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:private"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "graph_tensors_packet_generator_proto", srcs = ["graph_tensors_packet_generator.proto"], @@ -138,7 +138,7 @@ mediapipe_cc_proto_library( srcs = ["image_frame_to_tensor_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], visibility = ["//visibility:public"], deps = [":image_frame_to_tensor_calculator_proto"], @@ -173,7 +173,7 @@ mediapipe_cc_proto_library( srcs = ["pack_media_sequence_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], visibility = ["//visibility:public"], deps = [":pack_media_sequence_calculator_proto"], @@ -192,7 +192,7 @@ mediapipe_cc_proto_library( srcs = ["tensorflow_session_from_frozen_graph_generator.proto"], cc_deps = [ "//mediapipe/framework:packet_generator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], visibility = ["//visibility:public"], deps = [":tensorflow_session_from_frozen_graph_generator_proto"], @@ -203,7 +203,7 @@ mediapipe_cc_proto_library( srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], visibility = ["//visibility:public"], deps = [":tensorflow_session_from_frozen_graph_calculator_proto"], @@ -277,7 +277,7 @@ mediapipe_cc_proto_library( srcs = ["vector_int_to_tensor_calculator_options.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], visibility = ["//visibility:public"], deps = [":vector_int_to_tensor_calculator_options_proto"], @@ -296,7 +296,7 @@ cc_library( srcs = ["graph_tensors_packet_generator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto", + ":graph_tensors_packet_generator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -311,7 +311,7 @@ cc_library( srcs = ["image_frame_to_tensor_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:image_frame_to_tensor_calculator_cc_proto", + ":image_frame_to_tensor_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:ret_check", @@ -333,7 +333,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto", + ":matrix_to_tensor_calculator_options_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/port:status", @@ -354,7 +354,7 @@ cc_library( srcs = ["lapped_tensor_buffer_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto", + ":lapped_tensor_buffer_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -408,7 +408,7 @@ cc_library( "//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence_util", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], alwayslink = 1, ) @@ -423,7 +423,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], alwayslink = 1, ) @@ -436,7 +436,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":tensorflow_session", - "//mediapipe/calculators/tensorflow:tensorflow_inference_calculator_cc_proto", + ":tensorflow_inference_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/strings", @@ -514,7 +514,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":tensorflow_session", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto", + ":tensorflow_session_from_frozen_graph_generator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/tool:status_util", "//mediapipe/framework/port:status", @@ -573,7 +573,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":tensorflow_session", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto", + ":tensorflow_session_from_saved_model_generator_cc_proto", "//mediapipe/framework:packet_generator", "//mediapipe/framework:packet_type", "//mediapipe/framework/tool:status_util", @@ -597,7 +597,7 @@ cc_library( srcs = ["tensor_squeeze_dimensions_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto", + ":tensor_squeeze_dimensions_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -611,7 +611,7 @@ cc_library( srcs = ["tensor_to_image_frame_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto", + ":tensor_to_image_frame_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:ret_check", @@ -627,7 +627,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto", + ":tensor_to_matrix_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/port:status", @@ -654,7 +654,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], alwayslink = 1, ) @@ -667,7 +667,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:status", "//mediapipe/framework/port:ret_check", - "//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto", + ":tensor_to_vector_float_calculator_options_cc_proto", ] + select({ "//conditions:default": [ "@org_tensorflow//tensorflow/core:framework", @@ -695,7 +695,7 @@ cc_library( "//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], alwayslink = 1, ) @@ -719,7 +719,7 @@ cc_library( srcs = ["vector_float_to_tensor_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto", + ":vector_float_to_tensor_calculator_options_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -733,11 +733,11 @@ cc_library( srcs = ["unpack_yt8m_sequence_example_calculator.cc"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto", + ":lapped_tensor_buffer_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], alwayslink = 1, ) @@ -747,7 +747,7 @@ cc_test( srcs = ["graph_tensors_packet_generator_test.cc"], deps = [ ":graph_tensors_packet_generator", - "//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto", + ":graph_tensors_packet_generator_cc_proto", "//mediapipe/framework:packet", "//mediapipe/framework:packet_generator_cc_proto", "//mediapipe/framework:packet_set", @@ -779,7 +779,7 @@ cc_test( srcs = ["matrix_to_tensor_calculator_test.cc"], deps = [ ":matrix_to_tensor_calculator", - "//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto", + ":matrix_to_tensor_calculator_options_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/formats:matrix", @@ -795,13 +795,13 @@ cc_test( srcs = ["lapped_tensor_buffer_calculator_test.cc"], deps = [ ":lapped_tensor_buffer_calculator", - "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto", + ":lapped_tensor_buffer_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@com_google_absl//absl/memory", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -840,7 +840,7 @@ cc_test( "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -867,7 +867,7 @@ cc_test( "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:direct_session", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", "@org_tensorflow//tensorflow/core:testlib", "@org_tensorflow//tensorflow/core/kernels:conv_ops", "@org_tensorflow//tensorflow/core/kernels:math", @@ -883,7 +883,7 @@ cc_test( ":tensorflow_inference_calculator", ":tensorflow_session", ":tensorflow_session_from_frozen_graph_generator", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto", + ":tensorflow_session_from_frozen_graph_generator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework:packet_generator_cc_proto", @@ -897,7 +897,7 @@ cc_test( "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:direct_session", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", "@org_tensorflow//tensorflow/core:testlib", "@org_tensorflow//tensorflow/core/kernels:conv_ops", "@org_tensorflow//tensorflow/core/kernels:math", @@ -913,7 +913,7 @@ cc_test( ":tensorflow_inference_calculator", ":tensorflow_session", ":tensorflow_session_from_saved_model_generator", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto", + ":tensorflow_session_from_saved_model_generator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework:packet_generator_cc_proto", @@ -923,14 +923,8 @@ cc_test( "//mediapipe/framework/tool:tag_map_helper", "//mediapipe/framework/tool:validate_type", "@com_google_absl//absl/strings", + "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core/kernels:array", - "@org_tensorflow//tensorflow/core/kernels:bitcast_op", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core/kernels:io", - "@org_tensorflow//tensorflow/core/kernels:state", - "@org_tensorflow//tensorflow/core/kernels:string", - "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op", ], ) @@ -954,14 +948,8 @@ cc_test( "//mediapipe/framework/tool:tag_map_helper", "//mediapipe/framework/tool:validate_type", "@com_google_absl//absl/strings", + "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core/kernels:array", - "@org_tensorflow//tensorflow/core/kernels:bitcast_op", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core/kernels:io", - "@org_tensorflow//tensorflow/core/kernels:state", - "@org_tensorflow//tensorflow/core/kernels:string", - "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op", ], ) @@ -970,12 +958,12 @@ cc_test( srcs = ["tensor_squeeze_dimensions_calculator_test.cc"], deps = [ ":tensor_squeeze_dimensions_calculator", - "//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto", + ":tensor_squeeze_dimensions_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -985,13 +973,13 @@ cc_test( srcs = ["tensor_to_image_frame_calculator_test.cc"], deps = [ ":tensor_to_image_frame_calculator", - "//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto", + ":tensor_to_image_frame_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1001,14 +989,14 @@ cc_test( srcs = ["tensor_to_matrix_calculator_test.cc"], deps = [ ":tensor_to_matrix_calculator", - "//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto", + ":tensor_to_matrix_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1017,12 +1005,12 @@ cc_test( srcs = ["tensor_to_vector_float_calculator_test.cc"], deps = [ ":tensor_to_vector_float_calculator", - "//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto", + ":tensor_to_vector_float_calculator_options_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1042,7 +1030,7 @@ cc_test( "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1056,7 +1044,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1065,12 +1053,12 @@ cc_test( srcs = ["vector_float_to_tensor_calculator_test.cc"], deps = [ ":vector_float_to_tensor_calculator", - "//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto", + ":vector_float_to_tensor_calculator_options_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -1094,7 +1082,7 @@ cc_test( ":tensorflow_session", ":tensorflow_inference_calculator", ":tensorflow_session_from_frozen_graph_generator", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto", + ":tensorflow_session_from_frozen_graph_generator_cc_proto", "//mediapipe/framework/deps:file_path", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index a0b1fc0b..89b4d980 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -238,6 +238,7 @@ cc_library( "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", "@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert", "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", + "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal", ], "//conditions:default": [ "//mediapipe/gpu:gl_calculator_helper", diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index ebd632df..0abb4832 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -27,8 +27,7 @@ #include "tensorflow/lite/kernels/register.h" #include "tensorflow/lite/model.h" -#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \ - !defined(__APPLE__) +#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gpu_buffer.h" #include "tensorflow/lite/delegates/gpu/common/shape.h" @@ -49,6 +48,7 @@ #include "tensorflow/lite/delegates/gpu/common/shape.h" #include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h" #include "tensorflow/lite/delegates/gpu/metal_delegate.h" +#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h" #endif // iOS namespace { @@ -553,9 +553,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); #if defined(__APPLE__) && !TARGET_OS_OSX // iOS // Configure and create the delegate. - GpuDelegateOptions options; + TFLGpuDelegateOptions options; options.allow_precision_loss = false; // Must match converter, F=float/T=half - options.wait_type = GpuDelegateOptions::WaitType::kPassive; + options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive; if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options); id device = gpu_helper_.mtlDevice; diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc index 22d8b4d0..bac852f4 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc @@ -172,13 +172,11 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase { const int* detection_classes, std::vector* output_detections); Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, int class_id, - int detection_id, bool flip_vertically); + bool flip_vertically); int num_classes_ = 0; int num_boxes_ = 0; int num_coords_ = 0; - // Unique detection ID per new detection. - static int next_detection_id_; std::set ignore_classes_; ::mediapipe::TfLiteTensorsToDetectionsCalculatorOptions options_; @@ -199,10 +197,6 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase { }; REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator); -// Initialization of non-const static member should happen outside class -// definition. -int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0; - ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::GetContract( CalculatorContract* cc) { RET_CHECK(!cc->Inputs().GetTags().empty()); @@ -686,10 +680,7 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0; Detection detection = ConvertToDetection( detection_boxes[box_offset + 0], detection_boxes[box_offset + 1], detection_boxes[box_offset + 2], detection_boxes[box_offset + 3], - detection_scores[i], detection_classes[i], next_detection_id_, - options_.flip_vertically()); - // Increment to get next unique detection ID. - ++next_detection_id_; + detection_scores[i], detection_classes[i], options_.flip_vertically()); // Add keypoints. if (options_.num_keypoints() > 0) { auto* location_data = detection.mutable_location_data(); @@ -712,11 +703,10 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0; Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection( float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, - int class_id, int detection_id, bool flip_vertically) { + int class_id, bool flip_vertically) { Detection detection; detection.add_score(score); detection.add_label_id(class_id); - detection.set_detection_id(detection_id); LocationData* location_data = detection.mutable_location_data(); location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 9f3f687b..d3a2e341 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:public"]) exports_files(["LICENSE"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "annotation_overlay_calculator_proto", srcs = ["annotation_overlay_calculator.proto"], @@ -72,6 +72,24 @@ proto_library( ], ) +proto_library( + name = "collection_has_min_size_calculator_proto", + srcs = ["collection_has_min_size_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_proto", + ], +) + +proto_library( + name = "association_calculator_proto", + srcs = ["association_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_proto", + ], +) + mediapipe_cc_proto_library( name = "annotation_overlay_calculator_cc_proto", srcs = ["annotation_overlay_calculator.proto"], @@ -141,6 +159,26 @@ mediapipe_cc_proto_library( ], ) +mediapipe_cc_proto_library( + name = "collection_has_min_size_calculator_cc_proto", + srcs = ["collection_has_min_size_calculator.proto"], + cc_deps = [ + "//mediapipe/framework:calculator_cc_proto", + ], + visibility = ["//mediapipe:__subpackages__"], + deps = [":collection_has_min_size_calculator_proto"], +) + +mediapipe_cc_proto_library( + name = "association_calculator_cc_proto", + srcs = ["association_calculator.proto"], + cc_deps = [ + "//mediapipe/framework:calculator_cc_proto", + ], + visibility = ["//mediapipe:__subpackages__"], + deps = [":association_calculator_proto"], +) + cc_library( name = "packet_frequency_calculator", srcs = ["packet_frequency_calculator.cc"], @@ -847,3 +885,101 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "filter_collection_calculator", + srcs = ["filter_collection_calculator.cc"], + hdrs = ["filter_collection_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) + +cc_library( + name = "collection_has_min_size_calculator", + srcs = ["collection_has_min_size_calculator.cc"], + hdrs = ["collection_has_min_size_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + ":collection_has_min_size_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + +cc_library( + name = "association_calculator", + hdrs = ["association_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + ":association_calculator_cc_proto", + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", + "//mediapipe/framework/port:rectangle", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/memory", + ], + alwayslink = 1, +) + +cc_library( + name = "association_norm_rect_calculator", + srcs = ["association_norm_rect_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":association_calculator", + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:rectangle", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + +cc_library( + name = "association_detection_calculator", + srcs = ["association_detection_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":association_calculator", + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:location", + "//mediapipe/framework/port:rectangle", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + +cc_test( + name = "association_calculator_test", + srcs = ["association_calculator_test.cc"], + deps = [ + ":association_detection_calculator", + ":association_norm_rect_calculator", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework:collection_item_id", + "//mediapipe/framework:packet", + "//mediapipe/framework/deps:message_matchers", + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:location_data_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + ], +) diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index 4b8776c3..35341656 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -148,9 +148,6 @@ class AnnotationOverlayCalculator : public CalculatorBase { // Underlying helper renderer library. std::unique_ptr renderer_; - // Number of input streams with render data. - int num_render_streams_; - // Indicates if image frame is available as input. bool image_frame_available_ = false; @@ -181,20 +178,15 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator); return ::mediapipe::InternalError("GPU output must have GPU input."); } - // Assume all inputs are render streams; adjust below. - int num_render_streams = cc->Inputs().NumEntries(); - // Input image to render onto copy of. #if !defined(MEDIAPIPE_DISABLE_GPU) if (cc->Inputs().HasTag(kInputFrameTagGpu)) { cc->Inputs().Tag(kInputFrameTagGpu).Set(); - num_render_streams = cc->Inputs().NumEntries() - 1; use_gpu |= true; } #endif // !MEDIAPIPE_DISABLE_GPU if (cc->Inputs().HasTag(kInputFrameTag)) { cc->Inputs().Tag(kInputFrameTag).Set(); - num_render_streams = cc->Inputs().NumEntries() - 1; } // Data streams to render. @@ -246,12 +238,10 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator); if (cc->Inputs().HasTag(kInputFrameTagGpu) || cc->Inputs().HasTag(kInputFrameTag)) { image_frame_available_ = true; - num_render_streams_ = cc->Inputs().NumEntries() - 1; } else { image_frame_available_ = false; RET_CHECK(options_.has_canvas_width_px()); RET_CHECK(options_.has_canvas_height_px()); - num_render_streams_ = cc->Inputs().NumEntries(); } // Initialize the helper renderer library. diff --git a/mediapipe/calculators/util/association_calculator.h b/mediapipe/calculators/util/association_calculator.h new file mode 100644 index 00000000..a16de497 --- /dev/null +++ b/mediapipe/calculators/util/association_calculator.h @@ -0,0 +1,259 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ + +#include +#include + +#include "absl/memory/memory.h" +#include "mediapipe/calculators/util/association_calculator.pb.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/rectangle.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// Computes the overlap similarity based on Intersection over Union (IoU) of +// two rectangles. +inline float OverlapSimilarity(const Rectangle_f& rect1, + const Rectangle_f& rect2) { + if (!rect1.Intersects(rect2)) return 0.0f; + // Compute IoU similarity score. + const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area(); + const float normalization = rect1.Area() + rect2.Area() - intersection_area; + return normalization > 0.0f ? intersection_area / normalization : 0.0f; +} + +// AssocationCalculator accepts multiple inputs of vectors of type T that can +// be converted to Rectangle_f. The output is a vector of type T that contains +// elements from the input vectors that don't overlap with each other. When +// two elements overlap, the element that comes in from a later input stream +// is kept in the output. This association operation is useful for multiple +// instance inference pipelines in MediaPipe. +// If an input stream is tagged with "PREV" tag, IDs of overlapping elements +// from "PREV" input stream are propagated to the output. Elements in the "PREV" +// input stream that don't overlap with other elements are not added to the +// output. This stream is designed to take detections from previous timestamp, +// e.g. output of PreviousLoopbackCalculator to provide temporal association. +// See AssociationDetectionCalculator and AssociationNormRectCalculator for +// example uses. +template +class AssociationCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + // Atmost one input stream can be tagged with "PREV". + RET_CHECK_LE(cc->Inputs().NumEntries("PREV"), 1); + + if (cc->Inputs().HasTag("PREV")) { + RET_CHECK_GE(cc->Inputs().NumEntries(), 2); + } + + for (CollectionItemId id = cc->Inputs().BeginId(); + id < cc->Inputs().EndId(); ++id) { + cc->Inputs().Get(id).Set>(); + } + + cc->Outputs().Index(0).Set>(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + + has_prev_input_stream_ = cc->Inputs().HasTag("PREV"); + if (has_prev_input_stream_) { + prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0); + } + options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>(); + CHECK_GE(options_.min_similarity_threshold(), 0); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + auto get_non_overlapping_elements = GetNonOverlappingElements(cc); + if (!get_non_overlapping_elements.ok()) { + return get_non_overlapping_elements.status(); + } + std::list result = get_non_overlapping_elements.ValueOrDie(); + + if (has_prev_input_stream_ && + !cc->Inputs().Get(prev_input_stream_id_).IsEmpty()) { + // Processed all regular input streams. Now compare the result list + // elements with those in the PREV input stream, and propagate IDs from + // PREV input stream as appropriate. + const std::vector& prev_input_vec = + cc->Inputs() + .Get(prev_input_stream_id_) + .template Get>(); + + MP_RETURN_IF_ERROR( + PropagateIdsFromPreviousToCurrent(prev_input_vec, &result)); + } + + auto output = absl::make_unique>(); + for (auto it = result.begin(); it != result.end(); ++it) { + output->push_back(*it); + } + cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); + + return ::mediapipe::OkStatus(); + } + + protected: + ::mediapipe::AssociationCalculatorOptions options_; + + bool has_prev_input_stream_; + CollectionItemId prev_input_stream_id_; + + virtual ::mediapipe::StatusOr GetRectangle(const T& input) { + return ::mediapipe::OkStatus(); + } + + virtual std::pair GetId(const T& input) { return {false, -1}; } + + virtual void SetId(T* input, int id) {} + + private: + // Get a list of non-overlapping elements from all input streams, with + // increasing order of priority based on input stream index. + mediapipe::StatusOr> GetNonOverlappingElements( + CalculatorContext* cc) { + std::list result; + + // Initialize result with the first non-empty input vector. + CollectionItemId non_empty_id = cc->Inputs().BeginId(); + for (CollectionItemId id = cc->Inputs().BeginId(); + id < cc->Inputs().EndId(); ++id) { + if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) { + continue; + } + const std::vector& input_vec = + cc->Inputs().Get(id).Get>(); + if (!input_vec.empty()) { + non_empty_id = id; + result.push_back(input_vec[0]); + for (int j = 1; j < input_vec.size(); ++j) { + MP_RETURN_IF_ERROR(AddElementToList(input_vec[j], &result)); + } + break; + } + } + + // Compare remaining input vectors with the non-empty result vector, + // remove lower-priority overlapping elements from the result vector and + // had corresponding higher-priority elements as necessary. + for (CollectionItemId id = non_empty_id + 1; id < cc->Inputs().EndId(); + ++id) { + if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) { + continue; + } + const std::vector& input_vec = + cc->Inputs().Get(id).Get>(); + + for (int vi = 0; vi < input_vec.size(); ++vi) { + MP_RETURN_IF_ERROR(AddElementToList(input_vec[vi], &result)); + } + } + + return result; + } + + ::mediapipe::Status AddElementToList(T element, std::list* current) { + // Compare this element with elements of the input collection. If this + // element has high overlap with elements of the collection, remove + // those elements from the collection and add this element. + ASSIGN_OR_RETURN(auto cur_rect, GetRectangle(element)); + + bool change_id = false; + int new_elem_id = -1; + + for (auto uit = current->begin(); uit != current->end();) { + ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit)); + if (OverlapSimilarity(cur_rect, prev_rect) > + options_.min_similarity_threshold()) { + std::pair prev_id = GetId(*uit); + // If prev_id.first is false when some element doesn't have an ID, + // change_id and new_elem_id will not be updated. + if (prev_id.first) { + change_id = prev_id.first; + new_elem_id = prev_id.second; + } + uit = current->erase(uit); + } else { + ++uit; + } + } + + if (change_id) { + SetId(&element, new_elem_id); + } + current->push_back(element); + + return ::mediapipe::OkStatus(); + } + + // Compare elements of the current list with elements in from the collection + // of elements from the previous input stream, and propagate IDs from the + // previous input stream as appropriate. + ::mediapipe::Status PropagateIdsFromPreviousToCurrent( + const std::vector& prev_input_vec, std::list* current) { + for (auto vit = current->begin(); vit != current->end(); ++vit) { + auto get_cur_rectangle = GetRectangle(*vit); + if (!get_cur_rectangle.ok()) { + return get_cur_rectangle.status(); + } + const Rectangle_f& cur_rect = get_cur_rectangle.ValueOrDie(); + + bool change_id = false; + int id_for_vi = -1; + + for (int ui = 0; ui < prev_input_vec.size(); ++ui) { + auto get_prev_rectangle = GetRectangle(prev_input_vec[ui]); + if (!get_prev_rectangle.ok()) { + return get_prev_rectangle.status(); + } + const Rectangle_f& prev_rect = get_prev_rectangle.ValueOrDie(); + + if (OverlapSimilarity(cur_rect, prev_rect) > + options_.min_similarity_threshold()) { + std::pair prev_id = GetId(prev_input_vec[ui]); + // If prev_id.first is false when some element doesn't have an ID, + // change_id and id_for_vi will not be updated. + if (prev_id.first) { + change_id = prev_id.first; + id_for_vi = prev_id.second; + } + } + } + + if (change_id) { + T element = *vit; + SetId(&element, id_for_vi); + *vit = element; + } + } + return ::mediapipe::OkStatus(); + } +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/association_calculator.proto b/mediapipe/calculators/util/association_calculator.proto new file mode 100644 index 00000000..ca66f80b --- /dev/null +++ b/mediapipe/calculators/util/association_calculator.proto @@ -0,0 +1,27 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message AssociationCalculatorOptions { + extend CalculatorOptions { + optional AssociationCalculatorOptions ext = 275124847; + } + + optional float min_similarity_threshold = 1 [default = 1.0]; +} diff --git a/mediapipe/calculators/util/association_calculator_test.cc b/mediapipe/calculators/util/association_calculator_test.cc new file mode 100644 index 00000000..67fee7e8 --- /dev/null +++ b/mediapipe/calculators/util/association_calculator_test.cc @@ -0,0 +1,476 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/deps/message_matchers.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/location_data.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { + +namespace { + +::mediapipe::Detection DetectionWithRelativeLocationData(double xmin, + double ymin, + double width, + double height) { + ::mediapipe::Detection detection; + ::mediapipe::LocationData* location_data = detection.mutable_location_data(); + location_data->set_format(::mediapipe::LocationData::RELATIVE_BOUNDING_BOX); + location_data->mutable_relative_bounding_box()->set_xmin(xmin); + location_data->mutable_relative_bounding_box()->set_ymin(ymin); + location_data->mutable_relative_bounding_box()->set_width(width); + location_data->mutable_relative_bounding_box()->set_height(height); + return detection; +} + +} // namespace + +class AssociationDetectionCalculatorTest : public ::testing::Test { + protected: + AssociationDetectionCalculatorTest() { + // 0.4 ================ + // | | | | + // 0.3 ===================== | DET2 | | + // | | | DET1 | | | DET4 | + // 0.2 | DET0 | =========== ================ + // | | | | | | + // 0.1 =====|=============== | + // | DET3 | | | + // 0.0 ================ | + // | DET5 | + // -0.1 =========== + // 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2 + + // Detection det_0. + det_0 = DetectionWithRelativeLocationData(/*xmin=*/0.1, /*ymin=*/0.1, + /*width=*/0.2, /*height=*/0.2); + det_0.set_detection_id(0); + + // Detection det_1. + det_1 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/0.1, + /*width=*/0.2, /*height=*/0.2); + det_1.set_detection_id(1); + + // Detection det_2. + det_2 = DetectionWithRelativeLocationData(/*xmin=*/0.9, /*ymin=*/0.2, + /*width=*/0.2, /*height=*/0.2); + det_2.set_detection_id(2); + + // Detection det_3. + det_3 = DetectionWithRelativeLocationData(/*xmin=*/0.2, /*ymin=*/0.0, + /*width=*/0.3, /*height=*/0.3); + det_3.set_detection_id(3); + + // Detection det_4. + det_4 = DetectionWithRelativeLocationData(/*xmin=*/1.0, /*ymin=*/0.2, + /*width=*/0.2, /*height=*/0.2); + det_4.set_detection_id(4); + + // Detection det_5. + det_5 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/-0.1, + /*width=*/0.3, /*height=*/0.3); + det_5.set_detection_id(5); + } + + ::mediapipe::Detection det_0, det_1, det_2, det_3, det_4, det_5; +}; + +TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationDetectionCalculator" + input_stream: "input_vec_0" + input_stream: "input_vec_1" + input_stream: "input_vec_2" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream 0: det_0, det_1, det_2. + auto input_vec_0 = absl::make_unique>(); + input_vec_0->push_back(det_0); + input_vec_0->push_back(det_1); + input_vec_0->push_back(det_2); + runner.MutableInputs()->Index(0).packets.push_back( + Adopt(input_vec_0.release()).At(Timestamp(1))); + + // Input Stream 1: det_3, det_4. + auto input_vec_1 = absl::make_unique>(); + input_vec_1->push_back(det_3); + input_vec_1->push_back(det_4); + runner.MutableInputs()->Index(1).packets.push_back( + Adopt(input_vec_1.release()).At(Timestamp(1))); + + // Input Stream 2: det_5. + auto input_vec_2 = absl::make_unique>(); + input_vec_2->push_back(det_5); + runner.MutableInputs()->Index(2).packets.push_back( + Adopt(input_vec_2.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // det_3 overlaps with det_0, det_1 and det_5 overlaps with det_3. Since det_5 + // is in the highest priority, we remove other rects. det_4 overlaps with + // det_2, and det_4 is higher priority, so we keep it. The final output + // therefore contains 2 elements. + EXPECT_EQ(2, assoc_rects.size()); + // Outputs are in order of inputs, so det_4 is before det_5 in output vector. + + // det_4 overlaps with det_2, so new id for det_4 is 2. + EXPECT_TRUE(assoc_rects[0].has_detection_id()); + EXPECT_EQ(2, assoc_rects[0].detection_id()); + det_4.set_detection_id(2); + EXPECT_THAT(assoc_rects[0], EqualsProto(det_4)); + + // det_3 overlaps with det_0, so new id for det_3 is 0. + // det_3 overlaps with det_1, so new id for det_3 is 1. + // det_5 overlaps with det_3, so new id for det_5 is 1. + EXPECT_TRUE(assoc_rects[1].has_detection_id()); + EXPECT_EQ(1, assoc_rects[1].detection_id()); + det_5.set_detection_id(1); + EXPECT_THAT(assoc_rects[1], EqualsProto(det_5)); +} + +TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationDetectionCalculator" + input_stream: "PREV:input_vec_0" + input_stream: "input_vec_1" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream 0: det_3, det_4. + auto input_vec_0 = absl::make_unique>(); + input_vec_0->push_back(det_3); + input_vec_0->push_back(det_4); + CollectionItemId prev_input_stream_id = + runner.MutableInputs()->GetId("PREV", 0); + runner.MutableInputs() + ->Get(prev_input_stream_id) + .packets.push_back(Adopt(input_vec_0.release()).At(Timestamp(1))); + + // Input Stream 1: det_5. + auto input_vec_1 = absl::make_unique>(); + input_vec_1->push_back(det_5); + CollectionItemId input_stream_id = runner.MutableInputs()->GetId("", 0); + runner.MutableInputs() + ->Get(input_stream_id) + .packets.push_back(Adopt(input_vec_1.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // det_5 overlaps with det_3 and doesn't overlap with det_4. Since det_4 is + // in the PREV input stream, it doesn't get copied to the output, so the final + // output contains 1 element. + EXPECT_EQ(1, assoc_rects.size()); + + // det_5 overlaps with det_3, det_3 is in PREV, so new id for det_5 is 3. + EXPECT_TRUE(assoc_rects[0].has_detection_id()); + EXPECT_EQ(3, assoc_rects[0].detection_id()); + det_5.set_detection_id(3); + EXPECT_THAT(assoc_rects[0], EqualsProto(det_5)); +} + +TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationDetectionCalculator" + input_stream: "input_vec_0" + input_stream: "input_vec_1" + input_stream: "input_vec_2" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream 0: det_5. + auto input_vec_0 = absl::make_unique>(); + input_vec_0->push_back(det_5); + runner.MutableInputs()->Index(0).packets.push_back( + Adopt(input_vec_0.release()).At(Timestamp(1))); + + // Input Stream 1: det_3, det_4. + auto input_vec_1 = absl::make_unique>(); + input_vec_1->push_back(det_3); + input_vec_1->push_back(det_4); + runner.MutableInputs()->Index(1).packets.push_back( + Adopt(input_vec_1.release()).At(Timestamp(1))); + + // Input Stream 2: det_0, det_1, det_2. + auto input_vec_2 = absl::make_unique>(); + input_vec_2->push_back(det_0); + input_vec_2->push_back(det_1); + input_vec_2->push_back(det_2); + runner.MutableInputs()->Index(2).packets.push_back( + Adopt(input_vec_2.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // det_3 overlaps with det_5, so det_5 is removed. det_0 overlaps with det_3, + // so det_3 is removed as det_0 is in higher priority for keeping. det_2 + // overlaps with det_4 so det_4 is removed as det_2 is higher priority for + // keeping. The final output therefore contains 3 elements. + EXPECT_EQ(3, assoc_rects.size()); + // Outputs are in same order as inputs. + + // det_3 overlaps with det_5, so new id for det_3 is 5. + // det_0 overlaps with det_3, so new id for det_0 is 5. + EXPECT_TRUE(assoc_rects[0].has_detection_id()); + EXPECT_EQ(5, assoc_rects[0].detection_id()); + det_0.set_detection_id(5); + EXPECT_THAT(assoc_rects[0], EqualsProto(det_0)); + + // det_1 stays with id 1. + EXPECT_TRUE(assoc_rects[1].has_detection_id()); + EXPECT_EQ(1, assoc_rects[1].detection_id()); + EXPECT_THAT(assoc_rects[1], EqualsProto(det_1)); + + // det_2 overlaps with det_4, so new id for det_2 is 4. + EXPECT_TRUE(assoc_rects[2].has_detection_id()); + EXPECT_EQ(4, assoc_rects[2].detection_id()); + det_2.set_detection_id(4); + EXPECT_THAT(assoc_rects[2], EqualsProto(det_2)); +} + +class AssociationNormRectCalculatorTest : public ::testing::Test { + protected: + AssociationNormRectCalculatorTest() { + // 0.4 ================ + // | | | | + // 0.3 ===================== | NR2 | | + // | | | NR1 | | | NR4 | + // 0.2 | NR0 | =========== ================ + // | | | | | | + // 0.1 =====|=============== | + // | NR3 | | | + // 0.0 ================ | + // | NR5 | + // -0.1 =========== + // 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2 + + // NormalizedRect nr_0. + nr_0.set_x_center(0.2); + nr_0.set_y_center(0.2); + nr_0.set_width(0.2); + nr_0.set_height(0.2); + + // NormalizedRect nr_1. + nr_1.set_x_center(0.4); + nr_1.set_y_center(0.2); + nr_1.set_width(0.2); + nr_1.set_height(0.2); + + // NormalizedRect nr_2. + nr_2.set_x_center(1.0); + nr_2.set_y_center(0.3); + nr_2.set_width(0.2); + nr_2.set_height(0.2); + + // NormalizedRect nr_3. + nr_3.set_x_center(0.35); + nr_3.set_y_center(0.15); + nr_3.set_width(0.3); + nr_3.set_height(0.3); + + // NormalizedRect nr_4. + nr_4.set_x_center(1.1); + nr_4.set_y_center(0.3); + nr_4.set_width(0.2); + nr_4.set_height(0.2); + + // NormalizedRect nr_5. + nr_5.set_x_center(0.45); + nr_5.set_y_center(0.05); + nr_5.set_width(0.3); + nr_5.set_height(0.3); + } + + ::mediapipe::NormalizedRect nr_0, nr_1, nr_2, nr_3, nr_4, nr_5; +}; + +TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationNormRectCalculator" + input_stream: "input_vec_0" + input_stream: "input_vec_1" + input_stream: "input_vec_2" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream 0: nr_0, nr_1, nr_2. + auto input_vec_0 = + absl::make_unique>(); + input_vec_0->push_back(nr_0); + input_vec_0->push_back(nr_1); + input_vec_0->push_back(nr_2); + runner.MutableInputs()->Index(0).packets.push_back( + Adopt(input_vec_0.release()).At(Timestamp(1))); + + // Input Stream 1: nr_3, nr_4. + auto input_vec_1 = + absl::make_unique>(); + input_vec_1->push_back(nr_3); + input_vec_1->push_back(nr_4); + runner.MutableInputs()->Index(1).packets.push_back( + Adopt(input_vec_1.release()).At(Timestamp(1))); + + // Input Stream 2: nr_5. + auto input_vec_2 = + absl::make_unique>(); + input_vec_2->push_back(nr_5); + runner.MutableInputs()->Index(2).packets.push_back( + Adopt(input_vec_2.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // nr_3 overlaps with nr_0, nr_1 and nr_5 overlaps with nr_3. Since nr_5 is + // in the highest priority, we remove other rects. + // nr_4 overlaps with nr_2, and nr_4 is higher priority, so we keep it. + // The final output therefore contains 2 elements. + EXPECT_EQ(2, assoc_rects.size()); + // Outputs are in order of inputs, so nr_4 is before nr_5 in output vector. + EXPECT_THAT(assoc_rects[0], EqualsProto(nr_4)); + EXPECT_THAT(assoc_rects[1], EqualsProto(nr_5)); +} + +TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationNormRectCalculator" + input_stream: "input_vec_0" + input_stream: "input_vec_1" + input_stream: "input_vec_2" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream 0: nr_5. + auto input_vec_0 = + absl::make_unique>(); + input_vec_0->push_back(nr_5); + runner.MutableInputs()->Index(0).packets.push_back( + Adopt(input_vec_0.release()).At(Timestamp(1))); + + // Input Stream 1: nr_3, nr_4. + auto input_vec_1 = + absl::make_unique>(); + input_vec_1->push_back(nr_3); + input_vec_1->push_back(nr_4); + runner.MutableInputs()->Index(1).packets.push_back( + Adopt(input_vec_1.release()).At(Timestamp(1))); + + // Input Stream 2: nr_0, nr_1, nr_2. + auto input_vec_2 = + absl::make_unique>(); + input_vec_2->push_back(nr_0); + input_vec_2->push_back(nr_1); + input_vec_2->push_back(nr_2); + runner.MutableInputs()->Index(2).packets.push_back( + Adopt(input_vec_2.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // nr_3 overlaps with nr_5, so nr_5 is removed. nr_0 overlaps with nr_3, so + // nr_3 is removed as nr_0 is in higher priority for keeping. nr_2 overlaps + // with nr_4 so nr_4 is removed as nr_2 is higher priority for keeping. + // The final output therefore contains 3 elements. + EXPECT_EQ(3, assoc_rects.size()); + // Outputs are in same order as inputs. + EXPECT_THAT(assoc_rects[0], EqualsProto(nr_0)); + EXPECT_THAT(assoc_rects[1], EqualsProto(nr_1)); + EXPECT_THAT(assoc_rects[2], EqualsProto(nr_2)); +} + +TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) { + CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "AssociationNormRectCalculator" + input_stream: "input_vec" + output_stream: "output_vec" + options { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.1 + } + } + )")); + + // Input Stream : nr_3, nr_5. + auto input_vec = + absl::make_unique>(); + input_vec->push_back(nr_3); + input_vec->push_back(nr_5); + runner.MutableInputs()->Index(0).packets.push_back( + Adopt(input_vec.release()).At(Timestamp(1))); + + MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; + const std::vector& output = runner.Outputs().Index(0).packets; + EXPECT_EQ(1, output.size()); + const auto& assoc_rects = + output[0].Get>(); + + // nr_5 overlaps with nr_3. Since nr_5 is after nr_3 in the same input stream + // we remove nr_3 and keep nr_5. + // The final output therefore contains 1 elements. + EXPECT_EQ(1, assoc_rects.size()); + EXPECT_THAT(assoc_rects[0], EqualsProto(nr_5)); +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/association_detection_calculator.cc b/mediapipe/calculators/util/association_detection_calculator.cc new file mode 100644 index 00000000..125e8c4f --- /dev/null +++ b/mediapipe/calculators/util/association_detection_calculator.cc @@ -0,0 +1,77 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/association_calculator.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/location.h" +#include "mediapipe/framework/port/rectangle.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// A subclass of AssociationCalculator for Detection. Example: +// node { +// calculator: "AssociationDetectionCalculator" +// input_stream: "PREV:input_vec_0" +// input_stream: "input_vec_1" +// input_stream: "input_vec_2" +// output_stream: "output_vec" +// options { +// [mediapipe.AssociationCalculatorOptions.ext] { +// min_similarity_threshold: 0.1 +// } +// } +class AssociationDetectionCalculator + : public AssociationCalculator<::mediapipe::Detection> { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + return AssociationCalculator<::mediapipe::Detection>::GetContract(cc); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::Detection>::Open(cc); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::Detection>::Process(cc); + } + + ::mediapipe::Status Close(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::Detection>::Close(cc); + } + + protected: + ::mediapipe::StatusOr GetRectangle( + const ::mediapipe::Detection& input) override { + if (!input.has_location_data()) { + return ::mediapipe::InternalError("Missing location_data in Detection"); + } + const Location location(input.location_data()); + return location.GetRelativeBBox(); + } + + std::pair GetId(const ::mediapipe::Detection& input) override { + return {input.has_detection_id(), input.detection_id()}; + } + + void SetId(::mediapipe::Detection* input, int id) override { + input->set_detection_id(id); + } +}; + +REGISTER_CALCULATOR(AssociationDetectionCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/association_norm_rect_calculator.cc b/mediapipe/calculators/util/association_norm_rect_calculator.cc new file mode 100644 index 00000000..4069eda6 --- /dev/null +++ b/mediapipe/calculators/util/association_norm_rect_calculator.cc @@ -0,0 +1,72 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/association_calculator.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/rectangle.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// A subclass of AssociationCalculator for NormalizedRect. Example use case: +// node { +// calculator: "AssociationNormRectCalculator" +// input_stream: "input_vec_0" +// input_stream: "input_vec_1" +// input_stream: "input_vec_2" +// output_stream: "output_vec" +// options { +// [mediapipe.AssociationCalculatorOptions.ext] { +// min_similarity_threshold: 0.1 +// } +// } +class AssociationNormRectCalculator + : public AssociationCalculator<::mediapipe::NormalizedRect> { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + return AssociationCalculator<::mediapipe::NormalizedRect>::GetContract(cc); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::NormalizedRect>::Open(cc); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::NormalizedRect>::Process(cc); + } + + ::mediapipe::Status Close(CalculatorContext* cc) override { + return AssociationCalculator<::mediapipe::NormalizedRect>::Close(cc); + } + + protected: + ::mediapipe::StatusOr GetRectangle( + const ::mediapipe::NormalizedRect& input) override { + if (!input.has_x_center() || !input.has_y_center() || !input.has_width() || + !input.has_height()) { + return ::mediapipe::InternalError( + "Missing dimensions in NormalizedRect."); + } + const float xmin = input.x_center() - input.width() / 2.0; + const float ymin = input.y_center() - input.height() / 2.0; + // TODO: Support rotation for rectangle. + return Rectangle_f(xmin, ymin, input.width(), input.height()); + } +}; + +REGISTER_CALCULATOR(AssociationNormRectCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.cc b/mediapipe/calculators/util/collection_has_min_size_calculator.cc new file mode 100644 index 00000000..5ff43c60 --- /dev/null +++ b/mediapipe/calculators/util/collection_has_min_size_calculator.cc @@ -0,0 +1,26 @@ + +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/collection_has_min_size_calculator.h" + +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe { + +typedef CollectionHasMinSizeCalculator> + NormalizedRectVectorHasMinSizeCalculator; +REGISTER_CALCULATOR(NormalizedRectVectorHasMinSizeCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.h b/mediapipe/calculators/util/collection_has_min_size_calculator.h new file mode 100644 index 00000000..d8f0a282 --- /dev/null +++ b/mediapipe/calculators/util/collection_has_min_size_calculator.h @@ -0,0 +1,84 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ + +#include + +#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// Deterimines if an input iterable collection has a minimum size, specified +// in CollectionHasMinSizeCalculatorOptions. Example usage: +// node { +// calculator: "IntVectorHasMinSizeCalculator" +// input_stream: "ITERABLE:input_int_vector" +// output_stream: "has_min_ints" +// options { +// [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { +// min_size: 2 +// } +// } +// } +template +class CollectionHasMinSizeCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + RET_CHECK(cc->Inputs().HasTag("ITERABLE")); + RET_CHECK_EQ(1, cc->Inputs().NumEntries()); + + RET_CHECK_EQ(1, cc->Outputs().NumEntries()); + + RET_CHECK_GE( + cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>() + .min_size(), + 0); + + cc->Inputs().Tag("ITERABLE").Set(); + cc->Outputs().Index(0).Set(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + min_size_ = + cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>() + .min_size(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + const IterableT& input = cc->Inputs().Tag("ITERABLE").Get(); + bool has_min_size = input.size() >= min_size_; + + cc->Outputs().Index(0).AddPacket( + MakePacket(has_min_size).At(cc->InputTimestamp())); + + return ::mediapipe::OkStatus(); + } + + private: + int min_size_ = 0; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.proto b/mediapipe/calculators/util/collection_has_min_size_calculator.proto new file mode 100644 index 00000000..f482277c --- /dev/null +++ b/mediapipe/calculators/util/collection_has_min_size_calculator.proto @@ -0,0 +1,29 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message CollectionHasMinSizeCalculatorOptions { + extend CalculatorOptions { + optional CollectionHasMinSizeCalculatorOptions ext = 259397840; + } + + // The minimum size an input iterable collection should have for the + // calculator to output true. + optional int32 min_size = 1 [default = 0]; +} diff --git a/mediapipe/calculators/util/filter_collection_calculator.cc b/mediapipe/calculators/util/filter_collection_calculator.cc new file mode 100644 index 00000000..f86de04f --- /dev/null +++ b/mediapipe/calculators/util/filter_collection_calculator.cc @@ -0,0 +1,34 @@ + +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/filter_collection_calculator.h" + +#include + +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe { + +typedef FilterCollectionCalculator> + FilterNormalizedRectCollectionCalculator; +REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator); + +typedef FilterCollectionCalculator< + std::vector>> + FilterLandmarksCollectionCalculator; +REGISTER_CALCULATOR(FilterLandmarksCollectionCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/filter_collection_calculator.h b/mediapipe/calculators/util/filter_collection_calculator.h new file mode 100644 index 00000000..5f08dd98 --- /dev/null +++ b/mediapipe/calculators/util/filter_collection_calculator.h @@ -0,0 +1,109 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ + +#include + +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { + +// A calculator that gates elements of an input collection based on +// corresponding boolean values of the "CONDITION" vector. If there is no input +// collection or "CONDITION" vector, the calculator forwards timestamp bounds +// for downstream calculators. If the "CONDITION" vector has false values for +// all elements of the input collection, the calculator outputs a packet +// containing an empty collection. +// Example usage: +// node { +// calculator: "FilterCollectionCalculator" +// input_stream: "ITERABLE:input_collection" +// input_stream: "CONDITION:condition_vector" +// output_stream: "ITERABLE:output_collection" +// } +// This calculator is able to handle collections of copyable types T. +template +class FilterCollectionCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + RET_CHECK(cc->Inputs().HasTag("ITERABLE")); + RET_CHECK(cc->Inputs().HasTag("CONDITION")); + RET_CHECK(cc->Outputs().HasTag("ITERABLE")); + + cc->Inputs().Tag("ITERABLE").Set(); + cc->Inputs().Tag("CONDITION").Set>(); + + cc->Outputs().Tag("ITERABLE").Set(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + if (cc->Inputs().Tag("ITERABLE").IsEmpty()) { + return ::mediapipe::OkStatus(); + } + if (cc->Inputs().Tag("CONDITION").IsEmpty()) { + return ::mediapipe::OkStatus(); + } + + const std::vector& filter_by = + cc->Inputs().Tag("CONDITION").Get>(); + + return FilterCollection( + std::is_copy_constructible(), cc, + filter_by); + } + + template + ::mediapipe::Status FilterCollection(std::true_type, CalculatorContext* cc, + const std::vector& filter_by) { + const IterableU& input = cc->Inputs().Tag("ITERABLE").Get(); + if (input.size() != filter_by.size()) { + return ::mediapipe::InternalError(absl::StrCat( + "Input vector size: ", input.size(), + " doesn't mach condition vector size: ", filter_by.size())); + } + + auto output = absl::make_unique(); + for (int i = 0; i < input.size(); ++i) { + if (filter_by[i]) { + output->push_back(input[i]); + } + } + cc->Outputs().Tag("ITERABLE").Add(output.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } + + template + ::mediapipe::Status FilterCollection(std::false_type, CalculatorContext* cc, + const std::vector& filter_by) { + return ::mediapipe::InternalError( + "Cannot copy input collection to filter it."); + } +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/rect_transformation_calculator.cc b/mediapipe/calculators/util/rect_transformation_calculator.cc index 98a7da30..5b42a349 100644 --- a/mediapipe/calculators/util/rect_transformation_calculator.cc +++ b/mediapipe/calculators/util/rect_transformation_calculator.cc @@ -23,7 +23,9 @@ namespace mediapipe { namespace { constexpr char kNormRectTag[] = "NORM_RECT"; +constexpr char kNormRectsTag[] = "NORM_RECTS"; constexpr char kRectTag[] = "RECT"; +constexpr char kRectsTag[] = "RECTS"; constexpr char kImageSizeTag[] = "IMAGE_SIZE"; // Wraps around an angle in radians to within -M_PI and M_PI. @@ -72,17 +74,31 @@ REGISTER_CALCULATOR(RectTransformationCalculator); ::mediapipe::Status RectTransformationCalculator::GetContract( CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kNormRectTag) ^ cc->Inputs().HasTag(kRectTag)); + RET_CHECK_EQ((cc->Inputs().HasTag(kNormRectTag) ? 1 : 0) + + (cc->Inputs().HasTag(kNormRectsTag) ? 1 : 0) + + (cc->Inputs().HasTag(kRectTag) ? 1 : 0) + + (cc->Inputs().HasTag(kRectsTag) ? 1 : 0), + 1); if (cc->Inputs().HasTag(kRectTag)) { cc->Inputs().Tag(kRectTag).Set(); cc->Outputs().Index(0).Set(); } + if (cc->Inputs().HasTag(kRectsTag)) { + cc->Inputs().Tag(kRectsTag).Set>(); + cc->Outputs().Index(0).Set>(); + } if (cc->Inputs().HasTag(kNormRectTag)) { RET_CHECK(cc->Inputs().HasTag(kImageSizeTag)); cc->Inputs().Tag(kNormRectTag).Set(); cc->Inputs().Tag(kImageSizeTag).Set>(); cc->Outputs().Index(0).Set(); } + if (cc->Inputs().HasTag(kNormRectsTag)) { + RET_CHECK(cc->Inputs().HasTag(kImageSizeTag)); + cc->Inputs().Tag(kNormRectsTag).Set>(); + cc->Inputs().Tag(kImageSizeTag).Set>(); + cc->Outputs().Index(0).Set>(); + } return ::mediapipe::OkStatus(); } @@ -105,7 +121,17 @@ REGISTER_CALCULATOR(RectTransformationCalculator); cc->Outputs().Index(0).AddPacket( MakePacket(rect).At(cc->InputTimestamp())); } - + if (cc->Inputs().HasTag(kRectsTag) && + !cc->Inputs().Tag(kRectsTag).IsEmpty()) { + auto rects = cc->Inputs().Tag(kRectsTag).Get>(); + auto output_rects = absl::make_unique>(rects.size()); + for (int i = 0; i < rects.size(); ++i) { + output_rects->at(i) = rects[i]; + auto it = output_rects->begin() + i; + TransformRect(&(*it)); + } + cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp()); + } if (cc->Inputs().HasTag(kNormRectTag) && !cc->Inputs().Tag(kNormRectTag).IsEmpty()) { auto rect = cc->Inputs().Tag(kNormRectTag).Get(); @@ -115,6 +141,21 @@ REGISTER_CALCULATOR(RectTransformationCalculator); cc->Outputs().Index(0).AddPacket( MakePacket(rect).At(cc->InputTimestamp())); } + if (cc->Inputs().HasTag(kNormRectsTag) && + !cc->Inputs().Tag(kNormRectsTag).IsEmpty()) { + auto rects = + cc->Inputs().Tag(kNormRectsTag).Get>(); + const auto& image_size = + cc->Inputs().Tag(kImageSizeTag).Get>(); + auto output_rects = + absl::make_unique>(rects.size()); + for (int i = 0; i < rects.size(); ++i) { + output_rects->at(i) = rects[i]; + auto it = output_rects->begin() + i; + TransformNormalizedRect(&(*it), image_size.first, image_size.second); + } + cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp()); + } return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/video/BUILD b/mediapipe/calculators/video/BUILD index 63781a8b..d0f93a10 100644 --- a/mediapipe/calculators/video/BUILD +++ b/mediapipe/calculators/video/BUILD @@ -13,12 +13,16 @@ # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:private"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "flow_to_image_calculator_proto", srcs = ["flow_to_image_calculator.proto"], @@ -52,9 +56,7 @@ mediapipe_cc_proto_library( cc_library( name = "flow_to_image_calculator", srcs = ["flow_to_image_calculator.cc"], - visibility = [ - "//visibility:public", - ], + visibility = ["//visibility:public"], deps = [ "//mediapipe/calculators/video:flow_to_image_calculator_cc_proto", "//mediapipe/calculators/video/tool:flow_quantizer_model", @@ -129,10 +131,20 @@ cc_library( alwayslink = 1, ) +filegroup( + name = "test_videos", + srcs = [ + "testdata/format_FLV_H264_AAC.video", + "testdata/format_MKV_VP8_VORBIS.video", + "testdata/format_MP4_AVC720P_AAC.video", + ], + visibility = ["//visibility:public"], +) + cc_test( name = "opencv_video_decoder_calculator_test", srcs = ["opencv_video_decoder_calculator_test.cc"], - data = ["//mediapipe/calculators/video/testdata:test_videos"], + data = [":test_videos"], deps = [ ":opencv_video_decoder_calculator", "//mediapipe/framework:calculator_runner", @@ -151,7 +163,7 @@ cc_test( cc_test( name = "opencv_video_encoder_calculator_test", srcs = ["opencv_video_encoder_calculator_test.cc"], - data = ["//mediapipe/calculators/video/testdata:test_videos"], + data = [":test_videos"], deps = [ ":opencv_video_decoder_calculator", ":opencv_video_encoder_calculator", @@ -175,7 +187,6 @@ cc_test( cc_test( name = "tvl1_optical_flow_calculator_test", srcs = ["tvl1_optical_flow_calculator_test.cc"], - data = ["//mediapipe/calculators/image/testdata:test_images"], deps = [ ":tvl1_optical_flow_calculator", "//mediapipe/framework:calculator_framework", diff --git a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc index b333e3c7..49c7f055 100644 --- a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc +++ b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc @@ -123,6 +123,7 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase { cc->Outputs() .Tag("VIDEO_PRESTREAM") .Add(header.release(), Timestamp::PreStream()); + cc->Outputs().Tag("VIDEO_PRESTREAM").Close(); } // Rewind to the very first frame. cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0); diff --git a/mediapipe/calculators/video/testdata/BUILD b/mediapipe/calculators/video/testdata/BUILD deleted file mode 100644 index cd7c3d57..00000000 --- a/mediapipe/calculators/video/testdata/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -licenses(["notice"]) # Apache 2.0 - -filegroup( - name = "test_videos", - srcs = [ - "format_FLV_H264_AAC.video", - "format_MKV_VP8_VORBIS.video", - "format_MP4_AVC720P_AAC.video", - ], - visibility = ["//visibility:public"], -) diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD index 3d561c3a..96bc3566 100644 --- a/mediapipe/calculators/video/tool/BUILD +++ b/mediapipe/calculators/video/tool/BUILD @@ -13,24 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"]) exports_files(["LICENSE"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - proto_library( name = "flow_quantizer_model_proto", srcs = ["flow_quantizer_model.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], ) mediapipe_cc_proto_library( name = "flow_quantizer_model_cc_proto", srcs = ["flow_quantizer_model.proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":flow_quantizer_model_proto"], ) diff --git a/mediapipe/docs/examples.md b/mediapipe/docs/examples.md index e9d27e0f..4e279e93 100644 --- a/mediapipe/docs/examples.md +++ b/mediapipe/docs/examples.md @@ -76,6 +76,15 @@ MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline. * [Android](./hand_tracking_mobile_gpu.md) * [iOS](./hand_tracking_mobile_gpu.md) +### Multi-Hand Tracking with GPU + +[Multi-Hand Tracking with GPU](./multi_hand_tracking_mobile_gpu.md) illustrates +how to use MediaPipe with a TFLite model for multi-hand tracking in a +GPU-accelerated pipeline. + +* [Android](./multi_hand_tracking_mobile_gpu.md) +* [iOS](./multi_hand_tracking_mobile_gpu.md) + ### Hair Segmentation with GPU [Hair Segmentation on GPU](./hair_segmentation_mobile_gpu.md) illustrates how to @@ -132,6 +141,15 @@ with live video from a webcam. * [Desktop GPU](./hand_tracking_desktop.md) * [Desktop CPU](./hand_tracking_desktop.md) +### Multi-Hand Tracking on Desktop with Webcam + +[Multi-Hand Tracking on Desktop with Webcam](./multi_hand_tracking_desktop.md) +shows how to use MediaPipe with a TFLite model for multi-hand tracking on +desktop using CPU or GPU with live video from a webcam. + +* [Desktop GPU](./multi_hand_tracking_desktop.md) +* [Desktop CPU](./multi_hand_tracking_desktop.md) + ### Hair Segmentation on Desktop with Webcam [Hair Segmentation on Desktop with Webcam](./hair_segmentation_desktop.md) shows diff --git a/mediapipe/docs/face_detection_desktop.md b/mediapipe/docs/face_detection_desktop.md index f5e4c452..25caa2b7 100644 --- a/mediapipe/docs/face_detection_desktop.md +++ b/mediapipe/docs/face_detection_desktop.md @@ -77,7 +77,7 @@ below and paste it into ```bash # MediaPipe graph that performs face detection with TensorFlow Lite on CPU & GPU. # Used in the examples in -# mediapipie/examples/desktop/face_detection:face_detection_cpu. +# mediapipe/examples/desktop/face_detection:face_detection_cpu. # Images on CPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/docs/hair_segmentation_desktop.md b/mediapipe/docs/hair_segmentation_desktop.md index e5fd274d..50f92f8d 100644 --- a/mediapipe/docs/hair_segmentation_desktop.md +++ b/mediapipe/docs/hair_segmentation_desktop.md @@ -31,8 +31,6 @@ $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ #INFO: Found 1 target... #Target //mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu up-to-date: # bazel-bin/mediapipe/examples/desktop/hair_segmentation/hair_segmentation_gpu -#INFO: Elapsed time: 18.209s, Forge stats: 13026/13057 actions cached, 20.8s CPU used, 0.0s queue time, 89.3 MB ObjFS output (novel bytes: 87.4 MB), 0.0 MB local output, Critical Path: 11.88s, Remote (86.01% of the time): [queue: 0.00%, network: 16.83%, setup: 4.59%, process: 38.92%] -#INFO: Streaming build results to: http://sponge2/37d5a184-293b-4e98-a43e-b22084db3142 #INFO: Build completed successfully, 12210 total actions # This will open up your webcam as long as it is connected and on @@ -53,7 +51,7 @@ below and paste it into ```bash # MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU. # Used in the example in -# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. # Images on GPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/docs/hair_segmentation_mobile_gpu.md b/mediapipe/docs/hair_segmentation_mobile_gpu.md index 262d7a0c..602a2784 100644 --- a/mediapipe/docs/hair_segmentation_mobile_gpu.md +++ b/mediapipe/docs/hair_segmentation_mobile_gpu.md @@ -29,7 +29,7 @@ below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev/). ```bash # MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU. # Used in the example in -# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. # Images on GPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/docs/hand_tracking_desktop.md b/mediapipe/docs/hand_tracking_desktop.md index 3e8b10c8..f76b1dd9 100644 --- a/mediapipe/docs/hand_tracking_desktop.md +++ b/mediapipe/docs/hand_tracking_desktop.md @@ -31,8 +31,6 @@ $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \ # It should print: #Target //mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu up-to-date: # bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_cpu -#INFO: Elapsed time: 22.645s, Forge stats: 13356/13463 actions cached, 1.5m CPU used, 0.0s queue time, 819.8 MB ObjFS output (novel bytes: 85.6 MB), 0.0 MB local output, Critical Path: 14.43s, Remote (87.25% of the time): [queue: 0.00%, network: 14.88%, setup: 4.80%, process: 39.80%, fetch: 18.15%] -#INFO: Streaming build results to: http://sponge2/360196b9-33ab-44b1-84a7-1022b5043307 #INFO: Build completed successfully, 12517 total actions # This will open up your webcam as long as it is connected and on @@ -54,8 +52,6 @@ $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ # It should print: # Target //mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu up-to-date: # bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu -#INFO: Elapsed time: 84.055s, Forge stats: 6858/19343 actions cached, 1.6h CPU used, 0.9s queue time, 1.68 GB ObjFS output (novel bytes: 485.1 MB), 0.0 MB local output, Critical Path: 48.14s, Remote (99.40% of the time): [queue: 0.00%, setup: 5.59%, process: 74.44%] -#INFO: Streaming build results to: http://sponge2/00c7f95f-6fbc-432d-8978-f5d361efca3b #INFO: Build completed successfully, 22455 total actions # This will open up your webcam as long as it is connected and on @@ -77,7 +73,7 @@ below and paste it into # MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite # on CPU & GPU. # Used in the example in -# mediapipie/examples/desktop/hand_tracking:hand_tracking_cpu. +# mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu. # Images coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/docs/hand_tracking_mobile_gpu.md b/mediapipe/docs/hand_tracking_mobile_gpu.md index be9cdd26..e097dc9d 100644 --- a/mediapipe/docs/hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/hand_tracking_mobile_gpu.md @@ -100,8 +100,8 @@ see the Visualizing Subgraphs section in the ```bash # MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU. # Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and -# mediapipie/examples/ios/handtrackinggpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and +# mediapipe/examples/ios/handtrackinggpu. # Images coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/docs/images/mobile/multi_hand_detection_gpu_subgraph.png b/mediapipe/docs/images/mobile/multi_hand_detection_gpu_subgraph.png new file mode 100644 index 00000000..6105283b Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_detection_gpu_subgraph.png differ diff --git a/mediapipe/docs/images/mobile/multi_hand_landmark_subgraph.png b/mediapipe/docs/images/mobile/multi_hand_landmark_subgraph.png new file mode 100644 index 00000000..93f02bc4 Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_landmark_subgraph.png differ diff --git a/mediapipe/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png b/mediapipe/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png new file mode 100644 index 00000000..7da438e3 Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png differ diff --git a/mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif b/mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif new file mode 100644 index 00000000..fbdd8f57 Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif differ diff --git a/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu.gif b/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu.gif new file mode 100644 index 00000000..2cc920c8 Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu.gif differ diff --git a/mediapipe/docs/images/mobile/multi_hand_tracking_mobile.png b/mediapipe/docs/images/mobile/multi_hand_tracking_mobile.png new file mode 100644 index 00000000..b9eb410f Binary files /dev/null and b/mediapipe/docs/images/mobile/multi_hand_tracking_mobile.png differ diff --git a/mediapipe/docs/images/multi_hand_tracking_desktop.png b/mediapipe/docs/images/multi_hand_tracking_desktop.png new file mode 100644 index 00000000..5f84ab2f Binary files /dev/null and b/mediapipe/docs/images/multi_hand_tracking_desktop.png differ diff --git a/mediapipe/docs/install.md b/mediapipe/docs/install.md index fb3803cf..02b6fc14 100644 --- a/mediapipe/docs/install.md +++ b/mediapipe/docs/install.md @@ -7,11 +7,8 @@ future. Note: If you plan to use TensorFlow calculators and example apps, there is a known issue with gcc and g++ version 6.3 and 7.3. Please use other versions. -Note: While Mediapipe configures TensorFlow, if you see the -following error: -`"...git_configure.bzl", line 14, in _fail fail(("%sGit Configuration -Error:%s %...)))`, -please install the python future library using: `$ pip install --user future`. +Note: To make Mediapipe work with TensorFlow, please install the python "future" +library and the python "six" library using `pip install --user future six`. Choose your operating system: @@ -42,11 +39,19 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 0.24.1 and 0.29.1). +2. Install Bazel (0.24.1 and above required). - Follow the official - [documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) - to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet. + Option 1. Use package manager tool to install the latest version of Bazel. + + ```bash + $ sudo apt-get install bazel + + # Run 'bazel version' to check version of bazel installed + ``` + + Option 2. Follow the official + [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) + to install any version of Bazel manually. 3. Install OpenCV and FFmpeg. @@ -152,11 +157,11 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 0.24.1 and 0.29.1). +2. Install Bazel (0.24.1 and above required). Follow the official - [documentation](https://docs.bazel.build/versions/master/install-redhat.html) - to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet. + [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html) + to install Bazel manually. 3. Install OpenCV. @@ -240,24 +245,19 @@ To build and run iOS apps: $ cd mediapipe ``` -3. Install Bazel (version between 0.24.1 and 0.29.1). +3. Install Bazel (0.24.1 and above required). - Option 1. Use package manager tool to install Bazel 0.29.1 + Option 1. Use package manager tool to install the latest version of Bazel. ```bash - # If Bazel 1.0.0+ was installed. - $ brew uninstall bazel - - # Install Bazel 0.29.1 - $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/223ffb570c21c0a2af251afc6df9dec0214c6e74/Formula/bazel.rb - $ brew link bazel + $ brew install bazel # Run 'bazel version' to check version of bazel installed ``` Option 2. Follow the official - [documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x) - to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet. + [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x) + to install any version of Bazel manually. 4. Install OpenCV and FFmpeg. diff --git a/mediapipe/docs/multi_hand_tracking_desktop.md b/mediapipe/docs/multi_hand_tracking_desktop.md new file mode 100644 index 00000000..d80707f9 --- /dev/null +++ b/mediapipe/docs/multi_hand_tracking_desktop.md @@ -0,0 +1,177 @@ +## Multi-Hand Tracking on Desktop + +This is an example of using MediaPipe to run hand tracking models (TensorFlow +Lite) and render bounding boxes on the detected hand instances (for multiple +hands). To know more about the hand tracking models, please refer to the model +[`README file`]. Moreover, if you are interested in running the same TensorfFlow +Lite model on Android/iOS, please see the +[Mulit-Hand Tracking on GPU on Android/iOS](multi_hand_tracking_mobile_gpu.md) +and + +We show the hand tracking demos with TensorFlow Lite model using the Webcam: + +- [TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU)](#tensorflow-lite-multi-hand-tracking-demo-with-webcam-cpu) + +- [TensorFlow Lite Multi-Hand Tracking Demo with Webcam (GPU)](#tensorflow-lite-multi-hand-tracking-demo-with-webcam-gpu) + +Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please +see +[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md). + +Note: If MediaPipe depends on OpenCV 2, please see the +[known issues with OpenCV 2](#known-issues-with-opencv-2) section. + +### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU) + +To build and run the TensorFlow Lite example on desktop (CPU) with Webcam, run: + +```bash +# Video from webcam running on desktop CPU +$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \ + mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu + +# It should print: +#Target //mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu up-to-date: +# bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_cpu + +# This will open up your webcam as long as it is connected and on +# Any errors is likely due to your webcam being not accessible +$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_cpu \ + --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt +``` + +### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (GPU) + +To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run: + +```bash +# Video from webcam running on desktop GPU +# This works only for linux currently +$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ + mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu + +# It should print: +# Target //mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu up-to-date: +# bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_gpu + +# This will open up your webcam as long as it is connected and on +# Any errors is likely due to your webcam being not accessible, +# or GPU drivers not setup properly. +$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_gpu \ + --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt +``` + +#### Graph + +![graph visualization](images/multi_hand_tracking_desktop.png) + +To visualize the graph as shown above, copy the text specification of the graph +below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev). + +```bash +# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow +# Lite on CPU. +# Used in the example in +# mediapipie/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu. + +# Images coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided min_size. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks" + output_stream: "prev_has_enough_hands" + node_options: { + [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify max_vec_size in + # ClipVectorSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt + min_size: 2 + } + } +} + +# Drops the incoming image if the previous frame had at least N hands. +# Otherwise, passes the incoming image through to trigger a new round of hand +# detection in MultiHandDetectionSubgraph. +node { + calculator: "GateCalculator" + input_stream: "input_video" + input_stream: "DISALLOW:prev_has_enough_hands" + output_stream: "multi_hand_detection_input_video" + node_options: { + [type.googleapis.com/mediapipe.GateCalculatorOptions] { + empty_packets_as_allow: true + } + } +} + +# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt). +node { + calculator: "MultiHandDetectionSubgraph" + input_stream: "multi_hand_detection_input_video" + output_stream: "DETECTIONS:multi_palm_detections" + output_stream: "NORM_RECTS:multi_palm_rects" +} + +# Subgraph that localizes hand landmarks for multiple hands (see +# multi_hand_landmark.pbtxt). +node { + calculator: "MultiHandLandmarkSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks" +} + +# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the +# arrival of the next input image sends out the cached rectangle with the +# timestamp replaced by that of the input image, essentially generating a packet +# that carries the previous hand rectangle. Note that upon the arrival of the +# very first input image, an empty packet is sent out to jump start the +# feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:input_video" + input_stream: "LOOP:multi_hand_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks" +} + +# Performs association between NormalizedRect vector elements from previous +# frame and those from the current frame if MultiHandDetectionSubgraph runs. +# This calculator ensures that the output multi_hand_rects vector doesn't +# contain overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "prev_multi_hand_rects_from_landmarks" + input_stream: "multi_palm_rects" + output_stream: "multi_hand_rects" + node_options: { + [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { + min_similarity_threshold: 0.1 + } + } +} + +# Subgraph that renders annotations and overlays them on top of the input +# images (see multi_hand_renderer_cpu.pbtxt). +node { + calculator: "MultiHandRendererSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "DETECTIONS:multi_palm_detections" + input_stream: "LANDMARKS:multi_hand_landmarks" + input_stream: "NORM_RECTS:0:multi_palm_rects" + input_stream: "NORM_RECTS:1:multi_hand_rects" + output_stream: "IMAGE:output_video" +} +``` + +[`README file`]:https://github.com/google/mediapipe/tree/master/mediapipe/README.md diff --git a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md new file mode 100644 index 00000000..b57a6631 --- /dev/null +++ b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md @@ -0,0 +1,755 @@ +# Multi-Hand Tracking (GPU) + +This doc focuses on the +[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt) +that performs multi-hand tracking with TensorFlow Lite on GPU. It is related to +the [hand_tracking_example](./hand_tracking_mobile_gpu.md), and we recommend +users to review the (single) hand tracking example first. + +![multi_hand_tracking_android_gpu.gif](images/mobile/multi_hand_tracking_android_gpu.gif) + +In the visualization above, the red dots represent the hand landmarks and the +green lines are simply connections between selected landmark paris for +visualization of the hand skeleton. When there are fewer than `N` hands (`N=2` +in the graphs here), the purple box represents a hand rectangle that covers the +entire hand, derived from hand detection (see +[hand_detection_example](./hand_detection_mobile_gpu.md)). When there are `N` +hands (i.e. 2 hands for the graphs here), the red boxes represent hand +rectangles for each of the hands, derived from the previous round of hand +landmark localization using an ML model (see also +[model card](https://mediapipe.page.link/handmc)). Hand landmark localization +for each hand is performed only within the hand rectangle for computational +efficiency and accuracy. Hand detection is only invoked whenever there are fewer +than `N` hands in the previous iteration. + +This example can also run a model that localizes hand landmarks in 3D (i.e., +estimating an extra z coordinate): + +![multi_hand_tracking_3d_android_gpu.gif](images/mobile/multi_hand_tracking_3d_android_gpu.gif) + +In the visualization above, the localized hand landmarks are represented by dots +in different shades, with the brighter ones denoting landmarks closer to the +camera. + +## Android + +[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu) + +To build the app yourself, run: + +```bash +bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu +``` + +To build for the 3D mode, run: + +```bash +bazel build -c opt --config=android_arm64 --define 3D=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu +``` + +Once the app is built, install it on Android device with: + +```bash +adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/multihandtrackinggpu.apk +``` + +## iOS + +[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu). + +See the general [instructions](./mediapipe_ios_setup.md) for building iOS +examples and generating an Xcode project. This will be the HandDetectionGpuApp +target. + +To build on the command line: + +```bash +bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp +``` + +To build for the 3D mode, run: + +```bash +bazel build -c opt --config=ios_arm64 --define 3D=true mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp +``` + +## Graph + +The multi-hand tracking [main graph](#main-graph) internal utilizes a +[multi_hand_detection_subgraph](#multi-hand-detection-subgraph), a +[multi_hand_landmark_subgraph](#multi-hand-landmark-subgraph), and a +[multi_hand_renderer_subgraph](#multi-hand-renderer-subgraph). + +The subgraphs show up in the main graph visualization as nodes colored in +purple, and the subgraph itself can also be visualized just like a regular +graph. For more information on how to visualize a graph that includes subgraphs, +see the Visualizing Subgraphs section in the +[visualizer documentation](./visualizer.md). + +### Main Graph + +![multi_hand_tracking_mobile_graph](images/mobile/multi_hand_tracking_mobile.png) + +There are two key differences between this graph and the +[single_hand_tracking_mobile_graph](./hand_tracking_mobile_gpu.md). + +1. There is a `NormalizedRectVectorHasMinSize` calculator, that checks if in + input vector of `NormalizedRect` objects has a minimum size equal to `N`. In + this graph, if the vector contains fewer than `N` objects, + `MultiHandDetection` subgraph runs. Otherwise, the `GateCalculator` doesn't + send any image packets to the `MultiHandDetection` subgraph. This way, the + main graph is efficient in that it avoids running the costly hand detection + step when there are already `N` hands in the frame. +2. The `MergeCalculator` has been replaced by the `AssociationNormRect` + calculator. This `AssociationNormRect` takes as input a vector of + `NormalizedRect` objects from the `MultiHandDetection` subgraph on the + current frame, and a vector of `NormalizedRect` objects from the + `MultiHandLandmark` subgraph from the previous frame, and performs an + association operation between these objects. This calculator ensures that + the output vector doesn't contain overlapping regions based on the specified + `min_similarity_threshold`. + +[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt) + +```bash +# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU. +# Used in the examples in +# mediapipie/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu. + +# Images coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:multi_hand_rects" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided min_size. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks" + output_stream: "prev_has_enough_hands" + node_options: { + [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify max_vec_size in + # ClipVectorSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt + min_size: 2 + } + } +} + +# Drops the incoming image if the previous frame had at least N hands. +# Otherwise, passes the incoming image through to trigger a new round of hand +# detection in MultiHandDetectionSubgraph. +node { + calculator: "GateCalculator" + input_stream: "throttled_input_video" + input_stream: "DISALLOW:prev_has_enough_hands" + output_stream: "multi_hand_detection_input_video" + node_options: { + [type.googleapis.com/mediapipe.GateCalculatorOptions] { + empty_packets_as_allow: true + } + } +} + +# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt). +node { + calculator: "MultiHandDetectionSubgraph" + input_stream: "multi_hand_detection_input_video" + output_stream: "DETECTIONS:multi_palm_detections" + output_stream: "NORM_RECTS:multi_palm_rects" +} + +# Subgraph that localizes hand landmarks for multiple hands (see +# multi_hand_landmark.pbtxt). +node { + calculator: "MultiHandLandmarkSubgraph" + input_stream: "IMAGE:throttled_input_video" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks" +} + +# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the +# arrival of the next input image sends out the cached rectangle with the +# timestamp replaced by that of the input image, essentially generating a packet +# that carries the previous hand rectangle. Note that upon the arrival of the +# very first input image, an empty packet is sent out to jump start the +# feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:throttled_input_video" + input_stream: "LOOP:multi_hand_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks" +} + +# Performs association between NormalizedRect vector elements from previous +# frame and those from the current frame if MultiHandDetectionSubgraph runs. +# This calculator ensures that the output multi_hand_rects vector doesn't +# contain overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "prev_multi_hand_rects_from_landmarks" + input_stream: "multi_palm_rects" + output_stream: "multi_hand_rects" + node_options: { + [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { + min_similarity_threshold: 0.1 + } + } +} + +# Subgraph that renders annotations and overlays them on top of the input +# images (see multi_hand_renderer_gpu.pbtxt). +node { + calculator: "MultiHandRendererSubgraph" + input_stream: "IMAGE:throttled_input_video" + input_stream: "DETECTIONS:multi_palm_detections" + input_stream: "LANDMARKS:multi_hand_landmarks" + input_stream: "NORM_RECTS:0:multi_palm_rects" + input_stream: "NORM_RECTS:1:multi_hand_rects" + output_stream: "IMAGE:output_video" +} +``` + +### Multi-Hand Detection Subgraph + +![multi_hand_detection_gpu_subgraph](images/mobile/multi_hand_detection_gpu_subgraph.png) + +This graph outputs a vector of `NormalizedRect` objects corresponding to each of +the hand instances visible in the frame. Note that at the end of this graph, +there is a `ClipNormalizedRectVectorSizeCalculator`. This calculator clips the +size of the input vector to a maximum size `N`. This implies that the +`MultiHandDetection` subgraph outputs a vector of maximum `N` hand instance +locations. + +[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt) + +```bash +# MediaPipe multi-hand detection subgraph. + +type: "MultiHandDetectionSubgraph" + +input_stream: "input_video" +output_stream: "DETECTIONS:palm_detections" +output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections" + +# Transforms the input image on GPU to a 256x256 image. To scale the input +# image, the scale_mode option is set to FIT to preserve the aspect ratio, +# resulting in potential letterboxing in the transformed image. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE_GPU:input_video" + output_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "LETTERBOX_PADDING:letterbox_padding" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + output_width: 256 + output_height: 256 + scale_mode: FIT + } + } +} + +# Generates a single side packet containing a TensorFlow Lite op resolver that +# supports custom ops needed by the model used in this graph. +node { + calculator: "TfLiteCustomOpResolverCalculator" + output_side_packet: "opresolver" + node_options: { + [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] { + use_gpu: true + } + } +} + +# Converts the transformed input image on GPU into an image tensor stored as a +# TfLiteTensor. +node { + calculator: "TfLiteConverterCalculator" + input_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "TENSORS_GPU:image_tensor" +} + +# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS_GPU:image_tensor" + output_stream: "TENSORS_GPU:detection_tensors" + input_side_packet: "CUSTOM_OP_RESOLVER:opresolver" + node_options: { + [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { + model_path: "mediapipe/models/palm_detection.tflite" + use_gpu: true + } + } +} + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + node_options: { + [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { + num_layers: 5 + min_scale: 0.1171875 + max_scale: 0.75 + input_size_height: 256 + input_size_width: 256 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 8 + strides: 16 + strides: 32 + strides: 32 + strides: 32 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TfLiteTensorsToDetectionsCalculator" + input_stream: "TENSORS_GPU:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { + num_classes: 1 + num_boxes: 2944 + num_coords: 18 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 7 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + + x_scale: 256.0 + y_scale: 256.0 + h_scale: 256.0 + w_scale: 256.0 + min_score_thresh: 0.7 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + node_options: { + [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + return_empty_detections: true + } + } +} + +# Maps detection label IDs to the corresponding label text ("Palm"). The label +# map is provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "labeled_detections" + node_options: { + [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { + label_map_path: "mediapipe/models/palm_detection_labelmap.txt" + } + } +} + +# Adjusts detection locations (already normalized to [0.f, 1.f]) on the +# letterboxed image (after image transformation with the FIT scale mode) to the +# corresponding locations on the same image with the letterbox removed (the +# input image to the graph before image transformation). +node { + calculator: "DetectionLetterboxRemovalCalculator" + input_stream: "DETECTIONS:labeled_detections" + input_stream: "LETTERBOX_PADDING:letterbox_padding" + output_stream: "DETECTIONS:palm_detections" +} + +# Extracts image size from the input images. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:input_video" + output_stream: "SIZE:image_size" +} + +# Converts each palm detection into a rectangle (normalized by image size) +# that encloses the palm and is rotated such that the line connecting center of +# the wrist and MCP of the middle finger is aligned with the Y-axis of the +# rectangle. +node { + calculator: "DetectionsToRectsCalculator" + input_stream: "DETECTIONS:palm_detections" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "NORM_RECTS:palm_rects" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] { + rotation_vector_start_keypoint_index: 0 # Center of wrist. + rotation_vector_end_keypoint_index: 2 # MCP of middle finger. + rotation_vector_target_angle_degrees: 90 + output_zero_rect_for_empty_detections: true + } + } +} + +# Expands and shifts the rectangle that contains the palm so that it's likely +# to cover the entire hand. +node { + calculator: "RectTransformationCalculator" + input_stream: "NORM_RECTS:palm_rects" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] { + scale_x: 2.6 + scale_y: 2.6 + shift_y: -0.5 + square_long: true + } + } +} + +# Clips the size of the input vector to the provided max_vec_size. This +# determines the maximum number of hand instances this graph outputs. +# Note that the performance gain of clipping detections earlier in this graph is +# minimal because NMS will minimize overlapping detections and the number of +# detections isn't expected to exceed 5-10. +node { + calculator: "ClipNormalizedRectVectorSizeCalculator" + input_stream: "hand_rects_from_palm_detections" + output_stream: "clipped_hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify min_size in + # CollectionHsMinSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt and + # mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt. + max_vec_size: 2 + } + } +} +``` + +### Multi-Hand Landmark Subgraph + +![multi_hand_landmark_subgraph.pbtxt](images/mobile/multi_hand_landmark_subgraph.png) + +This graph accepts as input a vector of `NormalizedRect` objects, corresponding +the the region of each hand instance in the input image. For each +`NormalizedRect` object, the graph runs the existing `HandLandmark` subgraph and +collect the outputs of this subgraph into vectors. This is enabled by +`BeginLoop` and `EndLoop` calculators. + +The `BeginLoop` calculator accepts as input a packet containing an iterable +collection of elements. This calculator is templatized (see +[begin_loop_calculator.h](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/begin_loop_calculator.h)). +If the input packet arrived at a timestamp `ts`, this calculator outputs each +element in the collection at a fake timestamp `internal_ts`. At the end of the +collection, the calculator outputs the arrival timestamp `ts` in the output +stream tagged with `BATCH_END`. + +The nodes between the `BeginLoop` calculator and the corresponding `EndLoop` +calculator process individual packets at the fake timestamps `internal_ts`. +After each element is processed, it is sent to the `EndLoop` calculator (see +[end_loop_calculator.h](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/end_loop_calculator.h)), +which collects these elements in an output collection. The `EndLoop` calculator +listens for packets from the `BATCH_END` output stream of the `BeginLoop` +calculator. When the `BATCH_END` packet containing the real timestamp `ts` +arrives at the `EndLoop` calculator, the `EndLoop` calculator outputs a packet +containing the collection of processed elements at the real timestamp `ts`. + +In the multi-hand landmark subgraph, the `EndLoop` calculators collect the +output vector of hand landmarks per hand instance, the boolean values indicating +the presence of each hand and the `NormalizedRect` objects corresponding to the +regions surrounding each hand into vectors. + +Finally, based on the hand presence boolean value, the graph filters the +collections of hand landmarks and `NormalizdRect` objects corresponding to each +hand instance. + +[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt) + +```bash +# MediaPipe hand landmark localization subgraph. + +type: "MultiHandLandmarkSubgraph" + +input_stream: "IMAGE:input_video" +# A vector of NormalizedRect, one per each hand detected. +input_stream: "NORM_RECTS:multi_hand_rects" +# A vector of NormalizedLandmarks, one set per each hand. +output_stream: "LANDMARKS:filtered_multi_hand_landmarks" +# A vector of NormalizedRect, one per each hand. +output_stream: "NORM_RECTS:filtered_multi_hand_rects_for_next_frame" + +# Outputs each element of multi_hand_rects at a fake timestamp for the rest +# of the graph to process. Clones the input_video packet for each +# single_hand_rect at the fake timestamp. At the end of the loop, +# outputs the BATCH_END timestamp for downstream calculators to inform them +# that all elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:multi_hand_rects" + input_stream: "CLONE:input_video" + output_stream: "ITEM:single_hand_rect" + output_stream: "CLONE:input_video_cloned" + output_stream: "BATCH_END:single_hand_rect_timestamp" +} + +node { + calculator: "HandLandmarkSubgraph" + input_stream: "IMAGE:input_video_cloned" + input_stream: "NORM_RECT:single_hand_rect" + output_stream: "LANDMARKS:single_hand_landmarks" + output_stream: "NORM_RECT:single_hand_rect_from_landmarks" + output_stream: "PRESENCE:single_hand_presence" +} + +# Collects the boolean presence value for each single hand into a vector. Upon +# receiving the BATCH_END timestamp, outputs a vector of boolean values at the +# BATCH_END timestamp. +node { + calculator: "EndLoopBooleanCalculator" + input_stream: "ITEM:single_hand_presence" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_presence" +} + +# Collects a set of landmarks for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarksVectorCalculator" + input_stream: "ITEM:single_hand_landmarks" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_landmarks" +} + +# Collects a NormalizedRect for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:single_hand_rect_from_landmarks" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_rects_for_next_frame" +} + +# Filters the input vector of landmarks based on hand presence value for each +# hand. If the hand presence for hand #i is false, the set of landmarks +# corresponding to that hand are dropped from the vector. +node { + calculator: "FilterLandmarksCollectionCalculator" + input_stream: "ITERABLE:multi_hand_landmarks" + input_stream: "CONDITION:multi_hand_presence" + output_stream: "ITERABLE:filtered_multi_hand_landmarks" +} + +# Filters the input vector of NormalizedRect based on hand presence value for +# each hand. If the hand presence for hand #i is false, the NormalizedRect +# corresponding to that hand are dropped from the vector. +node { + calculator: "FilterNormalizedRectCollectionCalculator" + input_stream: "ITERABLE:multi_hand_rects_for_next_frame" + input_stream: "CONDITION:multi_hand_presence" + output_stream: "ITERABLE:filtered_multi_hand_rects_for_next_frame" +} +``` + +### Multi-Hand Renderer Subgraph + +![multi_hand_renderer_gpu_subgraph.pbtxt](images/mobile/multi_hand_renderer_gpu_subgraph.png) + +This graph also uses `BeginLoop` and `EndLoop` calculators to iteratively +convert a set of hand landmarks per hand instance into corresponding +`RenderData` objects. + +[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt) + +```bash +# MediaPipe multi-hand tracking rendering subgraph. + +type: "MultiHandRendererSubgraph" + +input_stream: "IMAGE:input_image" +# A vector of NormalizedLandmarks, one for each hand. +input_stream: "LANDMARKS:multi_hand_landmarks" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:0:multi_palm_rects" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:1:multi_hand_rects" +# A vector of Detection, one for each hand. +input_stream: "DETECTIONS:palm_detections" +output_stream: "IMAGE:output_image" + +# Converts detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:palm_detections" + output_stream: "RENDER_DATA:detection_render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 0 g: 255 b: 0 } + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "RENDER_DATA:multi_hand_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 255 g: 0 b: 0 } + thickness: 4.0 + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_palm_rects" + output_stream: "RENDER_DATA:multi_palm_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 125 g: 0 b: 122 } + thickness: 4.0 + } + } +} + +# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest +# of the graph to process. At the end of the loop, outputs the BATCH_END +# timestamp for downstream calculators to inform them that all elements in the +# vector have been processed. +node { + calculator: "BeginLoopNormalizedLandmarksVectorCalculator" + input_stream: "ITERABLE:multi_hand_landmarks" + output_stream: "ITEM:single_hand_landmarks" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to drawing primitives for annotation overlay. +node { + calculator: "LandmarksToRenderDataCalculator" + input_stream: "NORM_LANDMARKS:single_hand_landmarks" + output_stream: "RENDER_DATA:single_hand_landmark_render_data" + node_options: { + [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] { + landmark_connections: 0 + landmark_connections: 1 + landmark_connections: 1 + landmark_connections: 2 + landmark_connections: 2 + landmark_connections: 3 + landmark_connections: 3 + landmark_connections: 4 + landmark_connections: 0 + landmark_connections: 5 + landmark_connections: 5 + landmark_connections: 6 + landmark_connections: 6 + landmark_connections: 7 + landmark_connections: 7 + landmark_connections: 8 + landmark_connections: 5 + landmark_connections: 9 + landmark_connections: 9 + landmark_connections: 10 + landmark_connections: 10 + landmark_connections: 11 + landmark_connections: 11 + landmark_connections: 12 + landmark_connections: 9 + landmark_connections: 13 + landmark_connections: 13 + landmark_connections: 14 + landmark_connections: 14 + landmark_connections: 15 + landmark_connections: 15 + landmark_connections: 16 + landmark_connections: 13 + landmark_connections: 17 + landmark_connections: 0 + landmark_connections: 17 + landmark_connections: 17 + landmark_connections: 18 + landmark_connections: 18 + landmark_connections: 19 + landmark_connections: 19 + landmark_connections: 20 + landmark_color { r: 255 g: 0 b: 0 } + connection_color { r: 0 g: 255 b: 0 } + thickness: 4.0 + } + } +} + +# Collects a RenderData object for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END +# timestamp. +node { + calculator: "EndLoopRenderDataCalculator" + input_stream: "ITEM:single_hand_landmark_render_data" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_hand_landmarks_render_data" +} + +# Draws annotations and overlays them on top of the input images. Consumes +# a vector of RenderData objects and draws each of them on the input frame. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "detection_render_data" + input_stream: "multi_hand_rects_render_data" + input_stream: "multi_palm_rects_render_data" + input_stream: "VECTOR:0:multi_hand_landmarks_render_data" + output_stream: "OUTPUT_FRAME_GPU:output_image" +} +``` diff --git a/mediapipe/docs/object_detection_desktop.md b/mediapipe/docs/object_detection_desktop.md index ceb9da36..cb0b9011 100644 --- a/mediapipe/docs/object_detection_desktop.md +++ b/mediapipe/docs/object_detection_desktop.md @@ -54,7 +54,7 @@ below and paste it into # MediaPipe graph that performs object detection on desktop with TensorFlow # on CPU. # Used in the example in -# mediapipie/examples/desktop/object_detection:object_detection_tensorflow. +# mediapipe/examples/desktop/object_detection:object_detection_tensorflow. # Decodes an input video file into images and a video header. node { @@ -218,8 +218,6 @@ $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \ # It should print: #Target //mediapipe/examples/desktop/object_detection:object_detection_cpu up-to-date: # bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_cpu -#INFO: Elapsed time: 16.020s, Forge stats: 13001/13003 actions cached, 2.1s CPU used, 0.0s queue time, 89.0 MB ObjFS output (novel bytes: 88.0 MB), 0.0 MB local output, Critical Path: 10.01s, Remote (41.42% of the time): [queue: 0.00%, setup: 4.21%, process: 12.48%] -#INFO: Streaming build results to: http://sponge2/1824d4cc-ba63-4350-bdc0-aacbd45b902b #INFO: Build completed successfully, 12154 total actions # This will open up your webcam as long as it is connected and on @@ -240,7 +238,7 @@ below and paste it into # MediaPipe graph that performs object detection on desktop with TensorFlow Lite # on CPU. # Used in the example in -# mediapipie/examples/desktop/object_detection:object_detection_tflite. +# mediapipe/examples/desktop/object_detection:object_detection_tflite. # max_queue_size limits the number of packets enqueued on any input stream # by throttling inputs to the graph. This makes the graph only process one diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/AndroidManifest.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/AndroidManifest.xml new file mode 100644 index 00000000..bece0517 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/AndroidManifest.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD new file mode 100644 index 00000000..61c2065d --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD @@ -0,0 +1,103 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:private"]) + +cc_binary( + name = "libmediapipe_jni.so", + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators", + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + ], +) + +cc_library( + name = "mediapipe_jni_lib", + srcs = [":libmediapipe_jni.so"], + alwayslink = 1, +) + +# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be +# easily incorporated into the app via, for example, +# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb". +genrule( + name = "binary_graph", + srcs = ["//mediapipe/graphs/hand_tracking:multi_hand_tracking_mobile_gpu_binary_graph"], + outs = ["multihandtrackinggpu.binarypb"], + cmd = "cp $< $@", +) + +# To use the 3D model instead of the default 2D model, add "--define 3D=true" to the +# bazel build command. +config_setting( + name = "use_3d_model", + define_values = { + "3D": "true", + }, +) + +genrule( + name = "model", + srcs = select({ + "//conditions:default": ["//mediapipe/models:hand_landmark.tflite"], + ":use_3d_model": ["//mediapipe/models:hand_landmark_3d.tflite"], + }), + outs = ["hand_landmark.tflite"], + cmd = "cp $< $@", +) + +android_library( + name = "mediapipe_lib", + srcs = glob(["*.java"]), + assets = [ + ":binary_graph", + ":model", + "//mediapipe/models:palm_detection.tflite", + "//mediapipe/models:palm_detection_labelmap.txt", + ], + assets_dir = "", + manifest = "AndroidManifest.xml", + resource_files = glob(["res/**"]), + deps = [ + ":mediapipe_jni_lib", + "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper", + "//mediapipe/java/com/google/mediapipe/components:android_components", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/glutil", + "//third_party:androidx_appcompat", + "//third_party:androidx_constraint_layout", + "//third_party:androidx_legacy_support_v4", + "//third_party:androidx_material", + "//third_party:androidx_recyclerview", + "//third_party:opencv", + "@androidx_concurrent_futures//jar", + "@androidx_lifecycle//jar", + "@com_google_code_findbugs//jar", + "@com_google_guava_android//jar", + ], +) + +android_binary( + name = "multihandtrackinggpu", + manifest = "AndroidManifest.xml", + manifest_values = {"applicationId": "com.google.mediapipe.apps.multihandtrackinggpu"}, + multidex = "native", + deps = [ + ":mediapipe_lib", + ], +) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java new file mode 100644 index 00000000..cef13854 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java @@ -0,0 +1,167 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.apps.multihandtrackinggpu; + +import android.graphics.SurfaceTexture; +import android.os.Bundle; +import androidx.appcompat.app.AppCompatActivity; +import android.util.Size; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import android.view.View; +import android.view.ViewGroup; +import com.google.mediapipe.components.CameraHelper; +import com.google.mediapipe.components.CameraXPreviewHelper; +import com.google.mediapipe.components.ExternalTextureConverter; +import com.google.mediapipe.components.FrameProcessor; +import com.google.mediapipe.components.PermissionHelper; +import com.google.mediapipe.framework.AndroidAssetUtil; +import com.google.mediapipe.glutil.EglManager; + +/** Main activity of MediaPipe example apps. */ +public class MainActivity extends AppCompatActivity { + private static final String TAG = "MainActivity"; + + private static final String BINARY_GRAPH_NAME = "multihandtrackinggpu.binarypb"; + private static final String INPUT_VIDEO_STREAM_NAME = "input_video"; + private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video"; + private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT; + + // Flips the camera-preview frames vertically before sending them into FrameProcessor to be + // processed in a MediaPipe graph, and flips the processed frames back when they are displayed. + // This is needed because OpenGL represents images assuming the image origin is at the bottom-left + // corner, whereas MediaPipe in general assumes the image origin is at top-left. + private static final boolean FLIP_FRAMES_VERTICALLY = true; + + static { + // Load all native libraries needed by the app. + System.loadLibrary("mediapipe_jni"); + System.loadLibrary("opencv_java4"); + } + + // {@link SurfaceTexture} where the camera-preview frames can be accessed. + private SurfaceTexture previewFrameTexture; + // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph. + private SurfaceView previewDisplayView; + + // Creates and manages an {@link EGLContext}. + private EglManager eglManager; + // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed + // frames onto a {@link Surface}. + private FrameProcessor processor; + // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be + // consumed by {@link FrameProcessor} and the underlying MediaPipe graph. + private ExternalTextureConverter converter; + + // Handles camera access via the {@link CameraX} Jetpack support library. + private CameraXPreviewHelper cameraHelper; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + previewDisplayView = new SurfaceView(this); + setupPreviewDisplayView(); + + // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g., + // binary graphs. + AndroidAssetUtil.initializeNativeAssetManager(this); + + eglManager = new EglManager(null); + processor = + new FrameProcessor( + this, + eglManager.getNativeContext(), + BINARY_GRAPH_NAME, + INPUT_VIDEO_STREAM_NAME, + OUTPUT_VIDEO_STREAM_NAME); + processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY); + + PermissionHelper.checkAndRequestCameraPermissions(this); + } + + @Override + protected void onResume() { + super.onResume(); + converter = new ExternalTextureConverter(eglManager.getContext()); + converter.setFlipY(FLIP_FRAMES_VERTICALLY); + converter.setConsumer(processor); + if (PermissionHelper.cameraPermissionsGranted(this)) { + startCamera(); + } + } + + @Override + protected void onPause() { + super.onPause(); + converter.close(); + } + + @Override + public void onRequestPermissionsResult( + int requestCode, String[] permissions, int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults); + } + + private void setupPreviewDisplayView() { + previewDisplayView.setVisibility(View.GONE); + ViewGroup viewGroup = findViewById(R.id.preview_display_layout); + viewGroup.addView(previewDisplayView); + + previewDisplayView + .getHolder() + .addCallback( + new SurfaceHolder.Callback() { + @Override + public void surfaceCreated(SurfaceHolder holder) { + processor.getVideoSurfaceOutput().setSurface(holder.getSurface()); + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + // (Re-)Compute the ideal size of the camera-preview display (the area that the + // camera-preview frames get rendered onto, potentially with scaling and rotation) + // based on the size of the SurfaceView that contains the display. + Size viewSize = new Size(width, height); + Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize); + + // Connect the converter to the camera-preview frames as its input (via + // previewFrameTexture), and configure the output width and height as the computed + // display size. + converter.setSurfaceTextureAndAttachToGLContext( + previewFrameTexture, displaySize.getWidth(), displaySize.getHeight()); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + processor.getVideoSurfaceOutput().setSurface(null); + } + }); + } + + private void startCamera() { + cameraHelper = new CameraXPreviewHelper(); + cameraHelper.setOnCameraStartedListener( + surfaceTexture -> { + previewFrameTexture = surfaceTexture; + // Make the display view visible to start showing the preview. This triggers the + // SurfaceHolder.Callback added to (the holder of) previewDisplayView. + previewDisplayView.setVisibility(View.VISIBLE); + }); + cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null); + } +} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/layout/activity_main.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/layout/activity_main.xml new file mode 100644 index 00000000..c19d7e62 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/layout/activity_main.xml @@ -0,0 +1,20 @@ + + + + + + + diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/colors.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/colors.xml new file mode 100644 index 00000000..69b22338 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/colors.xml @@ -0,0 +1,6 @@ + + + #008577 + #00574B + #D81B60 + diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/strings.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/strings.xml new file mode 100644 index 00000000..b03b7c66 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/strings.xml @@ -0,0 +1,4 @@ + + Multi-Hand Tracking GPU + Please grant camera permissions. + diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/styles.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/styles.xml new file mode 100644 index 00000000..5885930d --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/styles.xml @@ -0,0 +1,11 @@ + + + + + + diff --git a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py index 83500a6f..1d7f5b69 100644 --- a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py +++ b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py @@ -68,13 +68,15 @@ import os import random import subprocess import sys +import tarfile import tempfile import urllib -import zipfile + from absl import app from absl import flags from absl import logging import tensorflow as tf + from mediapipe.util.sequence import media_sequence as ms CITATION = r"""@article{kay2017kinetics, @@ -84,21 +86,28 @@ CITATION = r"""@article{kay2017kinetics, year={2017}, url = {https://deepmind.com/research/open-source/kinetics}, }""" -ANNOTATION_URL = "https://storage.googleapis.com/deepmind-media/research/Kinetics_700.zip" +ANNOTATION_URL = "https://storage.googleapis.com/deepmind-media/Datasets/kinetics700.tar.gz" SECONDS_TO_MICROSECONDS = 1000000 GRAPHS = ["tvl1_flow_and_rgb_from_file.pbtxt"] FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow" SPLITS = { "train": { "shards": 1000, - "examples": 545317}, - "val": {"shards": 100, - "examples": 35000}, - "test": {"shards": 100, - "examples": 70000}, - "custom": {"csv": None, # Add a CSV for your own data here. - "shards": 1, # Change this number to increase sharding. - "examples": -1}, # Negative 1 allows any number of examples. + "examples": 541632 + }, + "validate": { + "shards": 100, + "examples": 34727 + }, + "test": { + "shards": 100, + "examples": 69347 + }, + "custom": { + "csv": None, # Add a CSV for your own data here. + "shards": 1, # Change this number to increase sharding. + "examples": -1 + }, # Negative 1 allows any number of examples. } NUM_CLASSES = 700 @@ -312,18 +321,16 @@ class Kinetics(object): logging.info("Downloading annotations.") paths = {} if download_labels_for_map: - zip_path = os.path.join(self.path_to_data, ANNOTATION_URL.split("/")[-1]) - if not tf.io.gfile.exists(zip_path): - urlretrieve(ANNOTATION_URL, zip_path) - with zipfile.ZipFile(zip_path) as annotations_zip: - annotations_zip.extractall(self.path_to_data) - for split in ["train", "test", "val"]: - zip_path = os.path.join(self.path_to_data, - "kinetics_700_%s.zip" % split) - csv_path = zip_path.replace(".zip", ".csv") + tar_path = os.path.join(self.path_to_data, ANNOTATION_URL.split("/")[-1]) + if not tf.io.gfile.exists(tar_path): + urlretrieve(ANNOTATION_URL, tar_path) + with tarfile.open(tar_path) as annotations_tar: + annotations_tar.extractall(self.path_to_data) + for split in ["train", "test", "validate"]: + csv_path = os.path.join(self.path_to_data, "kinetics700/%s.csv" % split) if not tf.io.gfile.exists(csv_path): - with zipfile.ZipFile(zip_path) as annotations_zip: - annotations_zip.extractall(self.path_to_data) + with tarfile.open(tar_path) as annotations_tar: + annotations_tar.extractall(self.path_to_data) paths[split] = csv_path for split, contents in SPLITS.items(): if "csv" in contents and contents["csv"]: diff --git a/mediapipe/examples/desktop/multi_hand_tracking/BUILD b/mediapipe/examples/desktop/multi_hand_tracking/BUILD new file mode 100644 index 00000000..f8313354 --- /dev/null +++ b/mediapipe/examples/desktop/multi_hand_tracking/BUILD @@ -0,0 +1,42 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//mediapipe/examples:__subpackages__"]) + +cc_binary( + name = "multi_hand_tracking_tflite", + deps = [ + "//mediapipe/examples/desktop:simple_run_graph_main", + "//mediapipe/graphs/hand_tracking:multi_hand_desktop_tflite_calculators", + ], +) + +cc_binary( + name = "multi_hand_tracking_cpu", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/hand_tracking:multi_hand_desktop_tflite_calculators", + ], +) + +# Linux only +cc_binary( + name = "multi_hand_tracking_gpu", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_gpu", + "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators", + ], +) diff --git a/mediapipe/examples/desktop/object_detection/BUILD b/mediapipe/examples/desktop/object_detection/BUILD index ee683206..66b6d569 100644 --- a/mediapipe/examples/desktop/object_detection/BUILD +++ b/mediapipe/examples/desktop/object_detection/BUILD @@ -16,51 +16,12 @@ licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//mediapipe/examples:__subpackages__"]) -cc_library( - name = "object_detection_tensorflow_deps", - deps = [ - "@org_tensorflow//tensorflow/c/kernels:bitcast_op", - "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core/kernels:argmax_op", - "@org_tensorflow//tensorflow/core/kernels:bias_op", - "@org_tensorflow//tensorflow/core/kernels:cast_op", - "@org_tensorflow//tensorflow/core/kernels:concat_op", - "@org_tensorflow//tensorflow/core/kernels:constant_op", - "@org_tensorflow//tensorflow/core/kernels:control_flow_ops", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core/kernels:cwise_op", - "@org_tensorflow//tensorflow/core/kernels:depthwise_conv_op", - "@org_tensorflow//tensorflow/core/kernels:fused_batch_norm_op", - "@org_tensorflow//tensorflow/core/kernels:gather_op", - "@org_tensorflow//tensorflow/core/kernels:identity_op", - "@org_tensorflow//tensorflow/core/kernels:logging_ops", - "@org_tensorflow//tensorflow/core/kernels:matmul_op", - "@org_tensorflow//tensorflow/core/kernels:non_max_suppression_op", - "@org_tensorflow//tensorflow/core/kernels:pack_op", - "@org_tensorflow//tensorflow/core/kernels:reduction_ops", - "@org_tensorflow//tensorflow/core/kernels:relu_op", - "@org_tensorflow//tensorflow/core/kernels:reshape_op", - "@org_tensorflow//tensorflow/core/kernels:resize_bilinear_op", - "@org_tensorflow//tensorflow/core/kernels:sequence_ops", - "@org_tensorflow//tensorflow/core/kernels:shape_ops", - "@org_tensorflow//tensorflow/core/kernels:slice_op", - "@org_tensorflow//tensorflow/core/kernels:split_op", - "@org_tensorflow//tensorflow/core/kernels:tensor_array_ops", - "@org_tensorflow//tensorflow/core/kernels:tile_ops", - "@org_tensorflow//tensorflow/core/kernels:topk_op", - "@org_tensorflow//tensorflow/core/kernels:transpose_op", - "@org_tensorflow//tensorflow/core/kernels:unpack_op", - "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op", - ], - alwayslink = 1, -) - cc_binary( name = "object_detection_tensorflow", deps = [ - ":object_detection_tensorflow_deps", "//mediapipe/examples/desktop:simple_run_graph_main", "//mediapipe/graphs/object_detection:desktop_tensorflow_calculators", + "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:direct_session", ], ) diff --git a/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.h b/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.h new file mode 100644 index 00000000..6b0377ef --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.h @@ -0,0 +1,21 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +@interface AppDelegate : UIResponder + +@property(strong, nonatomic) UIWindow *window; + +@end diff --git a/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.m b/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.m new file mode 100644 index 00000000..9e1b7ff0 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.m @@ -0,0 +1,59 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "AppDelegate.h" + +@interface AppDelegate () + +@end + +@implementation AppDelegate + +- (BOOL)application:(UIApplication *)application + didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + // Override point for customization after application launch. + return YES; +} + +- (void)applicationWillResignActive:(UIApplication *)application { + // Sent when the application is about to move from active to inactive state. This can occur for + // certain types of temporary interruptions (such as an incoming phone call or SMS message) or + // when the user quits the application and it begins the transition to the background state. Use + // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering + // callbacks. Games should use this method to pause the game. +} + +- (void)applicationDidEnterBackground:(UIApplication *)application { + // Use this method to release shared resources, save user data, invalidate timers, and store + // enough application state information to restore your application to its current state in case + // it is terminated later. If your application supports background execution, this method is + // called instead of applicationWillTerminate: when the user quits. +} + +- (void)applicationWillEnterForeground:(UIApplication *)application { + // Called as part of the transition from the background to the active state; here you can undo + // many of the changes made on entering the background. +} + +- (void)applicationDidBecomeActive:(UIApplication *)application { + // Restart any tasks that were paused (or not yet started) while the application was inactive. If + // the application was previously in the background, optionally refresh the user interface. +} + +- (void)applicationWillTerminate:(UIApplication *)application { + // Called when the application is about to terminate. Save data if appropriate. See also + // applicationDidEnterBackground:. +} + +@end diff --git a/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/AppIcon.appiconset/Contents.json b/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000..a1895a24 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1,99 @@ +{ + "images" : [ + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "20x20", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "29x29", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "29x29", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "40x40", + "scale" : "3x" + }, + { + "idiom" : "iphone", + "size" : "60x60", + "scale" : "2x" + }, + { + "idiom" : "iphone", + "size" : "60x60", + "scale" : "3x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "20x20", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "29x29", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "29x29", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "40x40", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "76x76", + "scale" : "1x" + }, + { + "idiom" : "ipad", + "size" : "76x76", + "scale" : "2x" + }, + { + "idiom" : "ipad", + "size" : "83.5x83.5", + "scale" : "2x" + }, + { + "idiom" : "ios-marketing", + "size" : "1024x1024", + "scale" : "1x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} + diff --git a/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/Contents.json b/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/Contents.json new file mode 100644 index 00000000..7afcdfaf --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/Contents.json @@ -0,0 +1,7 @@ +{ + "info" : { + "version" : 1, + "author" : "xcode" + } +} + diff --git a/mediapipe/examples/ios/multihandtrackinggpu/BUILD b/mediapipe/examples/ios/multihandtrackinggpu/BUILD new file mode 100644 index 00000000..edfd5bb5 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/BUILD @@ -0,0 +1,95 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache 2.0 + +MIN_IOS_VERSION = "10.0" + +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_application", +) + +# To use the 3D model instead of the default 2D model, add "--define 3D=true" to the +# bazel build command. +config_setting( + name = "use_3d_model", + define_values = { + "3D": "true", + }, +) + +genrule( + name = "model", + srcs = select({ + "//conditions:default": ["//mediapipe/models:hand_landmark.tflite"], + ":use_3d_model": ["//mediapipe/models:hand_landmark_3d.tflite"], + }), + outs = ["hand_landmark.tflite"], + cmd = "cp $< $@", +) + +ios_application( + name = "MultiHandTrackingGpuApp", + bundle_id = "com.google.mediapipe.MultiHandTrackingGpu", + families = [ + "iphone", + "ipad", + ], + infoplists = ["Info.plist"], + minimum_os_version = MIN_IOS_VERSION, + provisioning_profile = "//mediapipe/examples/ios:provisioning_profile", + deps = [ + ":MultiHandTrackingGpuAppLibrary", + "@ios_opencv//:OpencvFramework", + ], +) + +objc_library( + name = "MultiHandTrackingGpuAppLibrary", + srcs = [ + "AppDelegate.m", + "ViewController.mm", + "main.m", + ], + hdrs = [ + "AppDelegate.h", + "ViewController.h", + ], + data = [ + "Base.lproj/LaunchScreen.storyboard", + "Base.lproj/Main.storyboard", + ":model", + "//mediapipe/graphs/hand_tracking:multi_hand_tracking_mobile_gpu_binary_graph", + "//mediapipe/models:palm_detection.tflite", + "//mediapipe/models:palm_detection_labelmap.txt", + ], + sdk_frameworks = [ + "AVFoundation", + "CoreGraphics", + "CoreMedia", + "UIKit", + ], + deps = [ + "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/objc:mediapipe_input_sources_ios", + "//mediapipe/objc:mediapipe_layer_renderer", + ] + select({ + "//mediapipe:ios_i386": [], + "//mediapipe:ios_x86_64": [], + "//conditions:default": [ + "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators", + ], + }), +) diff --git a/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/LaunchScreen.storyboard b/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/LaunchScreen.storyboard new file mode 100644 index 00000000..bfa36129 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/LaunchScreen.storyboard @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/Main.storyboard b/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/Main.storyboard new file mode 100644 index 00000000..e3bd912a --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/Main.storyboard @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mediapipe/examples/ios/multihandtrackinggpu/Info.plist b/mediapipe/examples/ios/multihandtrackinggpu/Info.plist new file mode 100644 index 00000000..30db14c6 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/Info.plist @@ -0,0 +1,42 @@ + + + + + NSCameraUsageDescription + This app uses the camera to demonstrate live video processing. + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + + + diff --git a/mediapipe/examples/ios/multihandtrackinggpu/ViewController.h b/mediapipe/examples/ios/multihandtrackinggpu/ViewController.h new file mode 100644 index 00000000..e0a5a636 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/ViewController.h @@ -0,0 +1,19 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +@interface ViewController : UIViewController + +@end diff --git a/mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm b/mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm new file mode 100644 index 00000000..2d7c5d7a --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm @@ -0,0 +1,178 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "ViewController.h" + +#import "mediapipe/objc/MPPGraph.h" +#import "mediapipe/objc/MPPCameraInputSource.h" +#import "mediapipe/objc/MPPLayerRenderer.h" + +static NSString* const kGraphName = @"multi_hand_tracking_mobile_gpu"; + +static const char* kInputStream = "input_video"; +static const char* kOutputStream = "output_video"; +static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue"; + +@interface ViewController () + +// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and +// sent video frames on _videoQueue. +@property(nonatomic) MPPGraph* mediapipeGraph; + +@end + +@implementation ViewController { + /// Handles camera access via AVCaptureSession library. + MPPCameraInputSource* _cameraSource; + + /// Inform the user when camera is unavailable. + IBOutlet UILabel* _noCameraLabel; + /// Display the camera preview frames. + IBOutlet UIView* _liveView; + /// Render frames in a layer. + MPPLayerRenderer* _renderer; + + /// Process camera frames on this queue. + dispatch_queue_t _videoQueue; +} + +#pragma mark - Cleanup methods + +- (void)dealloc { + self.mediapipeGraph.delegate = nil; + [self.mediapipeGraph cancel]; + // Ignore errors since we're cleaning up. + [self.mediapipeGraph closeAllInputStreamsWithError:nil]; + [self.mediapipeGraph waitUntilDoneWithError:nil]; +} + +#pragma mark - MediaPipe graph methods + ++ (MPPGraph*)loadGraphFromResource:(NSString*)resource { + // Load the graph config resource. + NSError* configLoadError = nil; + NSBundle* bundle = [NSBundle bundleForClass:[self class]]; + if (!resource || resource.length == 0) { + return nil; + } + NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; + NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; + if (!data) { + NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); + return nil; + } + + // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. + mediapipe::CalculatorGraphConfig config; + config.ParseFromArray(data.bytes, data.length); + + // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. + MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; + [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer]; + return newGraph; +} + +#pragma mark - UIViewController methods + +- (void)viewDidLoad { + [super viewDidLoad]; + + _renderer = [[MPPLayerRenderer alloc] init]; + _renderer.layer.frame = _liveView.layer.bounds; + [_liveView.layer addSublayer:_renderer.layer]; + _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop; + // When using the front camera, mirror the input for a more natural look. + _renderer.mirrored = YES; + + dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0); + _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute); + + _cameraSource = [[MPPCameraInputSource alloc] init]; + [_cameraSource setDelegate:self queue:_videoQueue]; + _cameraSource.sessionPreset = AVCaptureSessionPresetHigh; + _cameraSource.cameraPosition = AVCaptureDevicePositionFront; + // The frame's native format is rotated with respect to the portrait orientation. + _cameraSource.orientation = AVCaptureVideoOrientationPortrait; + + self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName]; + self.mediapipeGraph.delegate = self; + // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing. + self.mediapipeGraph.maxFramesInFlight = 2; +} + +// In this application, there is only one ViewController which has no navigation to other view +// controllers, and there is only one View with live display showing the result of running the +// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph +// setup/teardown and camera start/stop logic should be updated appropriately in response to the +// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times +// depending on the application navigation flow in that case. +- (void)viewWillAppear:(BOOL)animated { + [super viewWillAppear:animated]; + + [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) { + if (granted) { + [self startGraphAndCamera]; + dispatch_async(dispatch_get_main_queue(), ^{ + _noCameraLabel.hidden = YES; + }); + } + }]; +} + +- (void)startGraphAndCamera { + // Start running self.mediapipeGraph. + NSError* error; + if (![self.mediapipeGraph startWithError:&error]) { + NSLog(@"Failed to start graph: %@", error); + } + + // Start fetching frames from the camera. + dispatch_async(_videoQueue, ^{ + [_cameraSource start]; + }); +} + +#pragma mark - MPPGraphDelegate methods + +// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer + fromStream:(const std::string&)streamName { + if (streamName == kOutputStream) { + // Display the captured image on the screen. + CVPixelBufferRetain(pixelBuffer); + dispatch_async(dispatch_get_main_queue(), ^{ + [_renderer renderPixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + }); + } +} + +#pragma mark - MPPInputSourceDelegate methods + +// Must be invoked on _videoQueue. +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer + timestamp:(CMTime)timestamp + fromSource:(MPPInputSource*)source { + if (source != _cameraSource) { + NSLog(@"Unknown source: %@", source); + return; + } + [self.mediapipeGraph sendPixelBuffer:imageBuffer + intoStream:kInputStream + packetType:MPPPacketTypePixelBuffer]; +} + +@end diff --git a/mediapipe/examples/ios/multihandtrackinggpu/main.m b/mediapipe/examples/ios/multihandtrackinggpu/main.m new file mode 100644 index 00000000..7ffe5ea5 --- /dev/null +++ b/mediapipe/examples/ios/multihandtrackinggpu/main.m @@ -0,0 +1,22 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "AppDelegate.h" + +int main(int argc, char * argv[]) { + @autoreleasepool { + return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class])); + } +} diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 1a273670..d5586ae9 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -14,13 +14,12 @@ # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_py_proto_library") + licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:private"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_py_proto_library") - package_group( name = "mediapipe_internal", packages = [ @@ -464,6 +463,8 @@ cc_library( "//mediapipe/framework:packet_generator_cc_proto", "//mediapipe/framework:status_handler_cc_proto", "//mediapipe/framework:thread_pool_executor_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", "//mediapipe/gpu:graph_support", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:fixed_array", @@ -1272,6 +1273,7 @@ cc_library( "//mediapipe/framework/tool:validate", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index ecc50cd1..32a790ad 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -24,6 +24,7 @@ #include #include "absl/container/fixed_array.h" +#include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -1017,8 +1018,8 @@ void CalculatorGraph::UpdateThrottledNodes(InputStreamManager* stream, // TODO Change the throttling code to use the index directly // rather than looking up a stream name. int node_index = validated_graph_->OutputStreamToNode(stream->Name()); - std::unordered_set owned_set; - const std::unordered_set* upstream_nodes; + absl::flat_hash_set owned_set; + const absl::flat_hash_set* upstream_nodes; if (node_index >= validated_graph_->CalculatorInfos().size()) { // TODO just create a NodeTypeInfo object for each virtual node. owned_set.insert(node_index); @@ -1100,10 +1101,10 @@ bool CalculatorGraph::UnthrottleSources() { // This is a sufficient because succesfully growing at least one full input // stream during each call to UnthrottleSources will eventually resolve // each deadlock. - std::unordered_set full_streams; + absl::flat_hash_set full_streams; { absl::MutexLock lock(&full_input_streams_mutex_); - for (std::unordered_set& s : full_input_streams_) { + for (absl::flat_hash_set& s : full_input_streams_) { if (!s.empty()) { full_streams.insert(s.begin(), s.end()); } diff --git a/mediapipe/framework/calculator_graph.h b/mediapipe/framework/calculator_graph.h index 0e8f2920..9662a81e 100644 --- a/mediapipe/framework/calculator_graph.h +++ b/mediapipe/framework/calculator_graph.h @@ -23,13 +23,13 @@ #include #include #include -#include -#include #include #include #include "absl/base/macros.h" #include "absl/container/fixed_array.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" @@ -579,18 +579,18 @@ class CalculatorGraph { // A node is scheduled only if this set is empty. Similarly, a packet // is added to a graph input stream only if this set is empty. // Note that this vector contains an unused entry for each non-source node. - std::vector> full_input_streams_ + std::vector> full_input_streams_ GUARDED_BY(full_input_streams_mutex_); // Maps stream names to graph input stream objects. - std::unordered_map> + absl::flat_hash_map> graph_input_streams_; // Maps graph input streams to their virtual node ids. - std::unordered_map graph_input_stream_node_ids_; + absl::flat_hash_map graph_input_stream_node_ids_; // Maps graph input streams to their max queue size. - std::unordered_map graph_input_stream_max_queue_size_; + absl::flat_hash_map graph_input_stream_max_queue_size_; // The factory for making counters associated with this graph. std::unique_ptr counter_factory_; diff --git a/mediapipe/framework/calculator_graph_side_packet_test.cc b/mediapipe/framework/calculator_graph_side_packet_test.cc index 01171a6c..fd78dc7d 100644 --- a/mediapipe/framework/calculator_graph_side_packet_test.cc +++ b/mediapipe/framework/calculator_graph_side_packet_test.cc @@ -68,6 +68,7 @@ class CountAndOutputSummarySidePacketInCloseCalculator : public CalculatorBase { } ::mediapipe::Status Close(CalculatorContext* cc) final { + absl::SleepFor(absl::Milliseconds(300)); // For GetOutputSidePacket test. cc->OutputSidePackets().Index(0).Set( MakePacket(count_).At(Timestamp::Unset())); return ::mediapipe::OkStatus(); diff --git a/mediapipe/framework/graph_validation_test.cc b/mediapipe/framework/graph_validation_test.cc index 98492b8d..73e856a0 100644 --- a/mediapipe/framework/graph_validation_test.cc +++ b/mediapipe/framework/graph_validation_test.cc @@ -32,7 +32,7 @@ namespace mediapipe { namespace { // Shows validation success for a graph and a subgraph. -TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) { +TEST(GraphValidationTest, InitializeGraphFromProtos) { auto config_1 = ParseTextProtoOrDie(R"( type: "PassThroughGraph" input_stream: "INPUT:stream_1" @@ -102,7 +102,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) { } // Shows validation failure due to an unregistered subgraph. -TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) { +TEST(GraphValidationTest, InitializeGraphFromLinker) { EXPECT_FALSE(SubgraphRegistry::IsRegistered("DubQuadTestSubgraph")); ValidatedGraphConfig builder_1; ::mediapipe::Status status_1 = @@ -114,7 +114,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) { } // Shows validation success for a graph and a template subgraph. -TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) { +TEST(GraphValidationTest, InitializeTemplateFromProtos) { mediapipe::tool::TemplateParser::Parser parser; CalculatorGraphTemplate config_1; CHECK(parser.ParseFromString(R"( @@ -210,5 +210,109 @@ TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) { )"))); } +// Shows passing validation of optional subgraph inputs and output streams. +TEST(GraphValidationTest, OptionalSubgraphStreams) { + // A subgraph defining two optional input streams + // and two optional output streams. + auto config_1 = ParseTextProtoOrDie(R"( + type: "PassThroughGraph" + input_stream: "INPUT:input_0" + input_stream: "INPUT:1:input_1" + output_stream: "OUTPUT:output_0" + output_stream: "OUTPUT:1:output_1" + node { + calculator: "PassThroughCalculator" + input_stream: "input_0" # Any Type. + input_stream: "input_1" # Any Type. + output_stream: "output_0" # Same as input. + } + )"); + + // An enclosing graph that specifies one of the two optional input streams + // and one of the two optional output streams. + auto config_2 = ParseTextProtoOrDie(R"( + input_stream: "INPUT:foo_in" + output_stream: "OUTPUT:foo_out" + node { + calculator: "PassThroughCalculator" + input_stream: "foo_in" # Any Type. + output_stream: "foo_bar" # Same as input. + } + node { + calculator: "PassThroughGraph" + input_stream: "INPUT:foo_bar" # Any Type. + output_stream: "OUTPUT:foo_out" # Same as input. + } + )"); + + GraphValidation validation_1; + MP_EXPECT_OK(validation_1.Validate({config_1, config_2}, {})); + CalculatorGraph graph_1; + MP_EXPECT_OK(graph_1.Initialize({config_1, config_2}, {})); + EXPECT_THAT( + graph_1.Config(), + + // The result includes only the requested input and output streams. + EqualsProto(::mediapipe::ParseTextProtoOrDie(R"( + input_stream: "INPUT:foo_in" + output_stream: "OUTPUT:foo_out" + node { + calculator: "PassThroughCalculator" + input_stream: "foo_in" + output_stream: "foo_bar" + } + node { + calculator: "PassThroughCalculator" + input_stream: "foo_bar" + output_stream: "foo_out" + } + executor {} + )"))); +} + +// Shows failing validation of optional subgraph inputs and output streams. +TEST(GraphValidationTest, OptionalSubgraphStreamsMismatched) { + // A subgraph defining two optional input streams + // and two optional output streams. + auto config_1 = ParseTextProtoOrDie(R"( + type: "PassThroughGraph" + input_stream: "INPUT:input_0" + input_stream: "INPUT:1:input_1" + output_stream: "OUTPUT:output_0" + output_stream: "OUTPUT:1:output_1" + node { + calculator: "PassThroughCalculator" + input_stream: "input_0" # Any Type. + input_stream: "input_1" # Any Type. + output_stream: "output_0" # Same as input. + } + )"); + + // An enclosing graph that specifies one of the two optional input streams + // and both of the two optional output streams. + auto config_2 = ParseTextProtoOrDie(R"( + input_stream: "INPUT:foo_in" + output_stream: "OUTPUT:foo_out" + node { + calculator: "PassThroughCalculator" + input_stream: "foo_in" # Any Type. + output_stream: "foo_bar" # Same as input. + } + node { + calculator: "PassThroughGraph" + input_stream: "INPUT:foo_bar" # Any Type. + input_stream: "INPUT:1:foo_bar" # Any Type. + output_stream: "OUTPUT:foo_out" # Same as input. + } + )"); + + GraphValidation validation_1; + mediapipe::Status status = validation_1.Validate({config_1, config_2}, {}); + ASSERT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument); + ASSERT_THAT(status.ToString(), + testing::HasSubstr( + "PassThroughCalculator must use matching tags and indexes")); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/packet.h b/mediapipe/framework/packet.h index 8564abce..11cfb5cc 100644 --- a/mediapipe/framework/packet.h +++ b/mediapipe/framework/packet.h @@ -85,7 +85,7 @@ class Packet { // given timestamp. Does not modify *this. Packet At(class Timestamp timestamp) const&; - // The rvalue reference overload of Packet's memeber function + // The rvalue reference overload of Packet's member function // Packet::At(class Timestamp). Moves *this to a new Packet and returns // the new Packet with the given timestamp. Packet At(class Timestamp timestamp) &&; diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index 188c22e5..18d46e1c 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -247,6 +247,26 @@ cc_library( ], ) +cc_library( + name = "opencv_features2d", + hdrs = ["opencv_features2d_inc.h"], + visibility = ["//visibility:public"], + deps = [ + ":opencv_core", + "//third_party:opencv", + ], +) + +cc_library( + name = "opencv_calib3d", + hdrs = ["opencv_calib3d_inc.h"], + visibility = ["//visibility:public"], + deps = [ + ":opencv_core", + "//third_party:opencv", + ], +) + cc_library( name = "parse_text_proto", hdrs = [ diff --git a/mediapipe/framework/port/opencv_calib3d_inc.h b/mediapipe/framework/port/opencv_calib3d_inc.h new file mode 100644 index 00000000..687e0ca6 --- /dev/null +++ b/mediapipe/framework/port/opencv_calib3d_inc.h @@ -0,0 +1,26 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_ +#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_ + +#include + +#ifdef CV_VERSION_EPOCH // for OpenCV 2.x +#include +#else +#include +#endif + +#endif // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_ diff --git a/mediapipe/framework/port/opencv_features2d_inc.h b/mediapipe/framework/port/opencv_features2d_inc.h new file mode 100644 index 00000000..31ab2457 --- /dev/null +++ b/mediapipe/framework/port/opencv_features2d_inc.h @@ -0,0 +1,26 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_ +#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_ + +#include + +#ifdef CV_VERSION_EPOCH // for OpenCV 2.x +#include +#else +#include +#endif + +#endif // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_ diff --git a/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc index dad849c1..9e874c05 100644 --- a/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc @@ -155,7 +155,7 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler { return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one); } - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) { + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { DCHECK(min_stream_timestamp); absl::MutexLock lock(&erase_mutex_); // kReadyForProcess is returned only once until FillInputSet completes. diff --git a/mediapipe/framework/testdata/BUILD b/mediapipe/framework/testdata/BUILD index 59957689..0b96502c 100644 --- a/mediapipe/framework/testdata/BUILD +++ b/mediapipe/framework/testdata/BUILD @@ -31,7 +31,7 @@ mediapipe_cc_proto_library( name = "sky_light_calculator_cc_proto", srcs = ["sky_light_calculator.proto"], cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":sky_light_calculator_proto"], ) @@ -45,7 +45,7 @@ mediapipe_cc_proto_library( name = "night_light_calculator_cc_proto", srcs = ["night_light_calculator.proto"], cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//mediapipe:__subpackages__"], + visibility = ["//visibility:public"], deps = [":night_light_calculator_proto"], ) diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 06d178cc..61ec92e1 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -177,7 +177,7 @@ cc_library( deps = [ "//mediapipe/framework:packet", "//mediapipe/framework/port:statusor", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) diff --git a/mediapipe/framework/tool/mediapipe_graph.bzl b/mediapipe/framework/tool/mediapipe_graph.bzl index d6e7c56a..564fb301 100644 --- a/mediapipe/framework/tool/mediapipe_graph.bzl +++ b/mediapipe/framework/tool/mediapipe_graph.bzl @@ -110,7 +110,7 @@ def mediapipe_simple_subgraph( testonly: pass 1 if the graph is to be used only for tests. **kwargs: Remaining keyword args, forwarded to cc_library. """ - graph_base_name = graph.replace(":", "/").split("/")[-1].rsplit(".", 1)[0] + graph_base_name = name mediapipe_binary_graph( name = name + "_graph", graph = graph, diff --git a/mediapipe/framework/tool/subgraph_expansion.cc b/mediapipe/framework/tool/subgraph_expansion.cc index 665fd4ce..ddd8a06e 100644 --- a/mediapipe/framework/tool/subgraph_expansion.cc +++ b/mediapipe/framework/tool/subgraph_expansion.cc @@ -52,6 +52,31 @@ namespace tool { return ::mediapipe::OkStatus(); } +// Returns subgraph streams not requested by a subgraph-node. +::mediapipe::Status FindIgnoredStreams( + const proto_ns::RepeatedPtrField& src_streams, + const proto_ns::RepeatedPtrField& dst_streams, + std::set* result) { + ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(src_streams)); + ASSIGN_OR_RETURN(auto dst_map, tool::TagMap::Create(dst_streams)); + std::set_difference(src_map->Names().begin(), src_map->Names().end(), + dst_map->Names().begin(), dst_map->Names().end(), + std::inserter(*result, result->begin())); + return ::mediapipe::OkStatus(); +} + +// Removes subgraph streams not requested by a subgraph-node. +::mediapipe::Status RemoveIgnoredStreams( + proto_ns::RepeatedPtrField* streams, + const std::set& missing_streams) { + for (int i = streams->size() - 1; i >= 0; --i) { + if (missing_streams.count(streams->Get(i)) > 0) { + streams->DeleteSubrange(i, 1); + } + } + return ::mediapipe::OkStatus(); +} + ::mediapipe::Status TransformNames( CalculatorGraphConfig* config, const std::function& transform) { @@ -190,6 +215,14 @@ static ::mediapipe::Status PrefixNames(int subgraph_index, .SetPrepend() << "while processing the output side packets of subgraph node " << subgraph_node.calculator() << ": "; + std::set ignored_input_streams; + MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_stream(), + subgraph_node.input_stream(), + &ignored_input_streams)); + std::set ignored_input_side_packets; + MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_side_packet(), + subgraph_node.input_side_packet(), + &ignored_input_side_packets)); std::map* name_map; auto replace_names = [&name_map](absl::string_view s) { std::string original(s); @@ -207,6 +240,12 @@ static ::mediapipe::Status PrefixNames(int subgraph_index, TransformStreamNames(node.mutable_input_side_packet(), replace_names)); MP_RETURN_IF_ERROR( TransformStreamNames(node.mutable_output_side_packet(), replace_names)); + + // Remove input streams and side packets ignored by the subgraph-node. + MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_stream(), + ignored_input_streams)); + MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_side_packet(), + ignored_input_side_packets)); } name_map = &side_packet_map; for (auto& generator : *subgraph_config->mutable_packet_generator()) { diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index c9b2914e..a710cdb2 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -14,8 +14,7 @@ #include "mediapipe/framework/validated_graph_config.h" -#include - +#include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -934,7 +933,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex( } ::mediapipe::Status ValidatedGraphConfig::ValidateExecutors() { - std::unordered_set declared_names; + absl::flat_hash_set declared_names; for (const ExecutorConfig& executor_config : config_.executor()) { if (IsReservedExecutorName(executor_config.name())) { return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) @@ -964,7 +963,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex( << "\"" << executor_name << "\" is a reserved executor name."; } // The executor must be declared in an ExecutorConfig. - if (declared_names.find(executor_name) == declared_names.end()) { + if (!declared_names.contains(executor_name)) { return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) << "The executor \"" << executor_name << "\" is not declared in an ExecutorConfig."; diff --git a/mediapipe/framework/validated_graph_config.h b/mediapipe/framework/validated_graph_config.h index d7a84795..9554b71d 100644 --- a/mediapipe/framework/validated_graph_config.h +++ b/mediapipe/framework/validated_graph_config.h @@ -16,9 +16,9 @@ #define MEDIAPIPE_FRAMEWORK_VALIDATED_GRAPH_CONFIG_H_ #include -#include #include +#include "absl/container/flat_hash_set.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/packet_generator.pb.h" @@ -169,7 +169,7 @@ class NodeTypeInfo { // be a virtual node corresponding to a graph input stream (which are // listed by index contiguously after all calculators). // This function is only valid for a NodeTypeInfo of NodeType CALCULATOR. - const std::unordered_set& AncestorSources() const { + const absl::flat_hash_set& AncestorSources() const { return ancestor_sources_; } // Returns True if the source was not already there. @@ -213,7 +213,7 @@ class NodeTypeInfo { NodeRef node_; // The set of sources which affect this node. - std::unordered_set ancestor_sources_; + absl::flat_hash_set ancestor_sources_; }; // Information for either the input or output side of an edge. An edge diff --git a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt index 95fdb362..2d22b7a1 100644 --- a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt @@ -1,6 +1,6 @@ # MediaPipe graph that performs face detection with TensorFlow Lite on CPU. # Used in the examples in -# mediapipie/examples/desktop/face_detection:face_detection_cpu. +# mediapipe/examples/desktop/face_detection:face_detection_cpu. # Images on GPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/hand_tracking/BUILD b/mediapipe/graphs/hand_tracking/BUILD index 09a8e4d0..da6776b8 100644 --- a/mediapipe/graphs/hand_tracking/BUILD +++ b/mediapipe/graphs/hand_tracking/BUILD @@ -12,26 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -licenses(["notice"]) # Apache 2.0 - -package(default_visibility = ["//visibility:public"]) - load( "//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_binary_graph", ) +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + cc_library( - name = "desktop_tflite_calculators", + name = "desktop_offline_calculators", deps = [ "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/calculators/core:gate_calculator", "//mediapipe/calculators/core:immediate_mux_calculator", - "//mediapipe/calculators/core:merge_calculator", "//mediapipe/calculators/core:packet_inner_join_calculator", "//mediapipe/calculators/core:previous_loopback_calculator", "//mediapipe/calculators/video:opencv_video_decoder_calculator", "//mediapipe/calculators/video:opencv_video_encoder_calculator", + ], +) + +cc_library( + name = "desktop_tflite_calculators", + deps = [ + ":desktop_offline_calculators", + "//mediapipe/calculators/core:merge_calculator", "//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_cpu", "//mediapipe/graphs/hand_tracking/subgraphs:hand_landmark_cpu", "//mediapipe/graphs/hand_tracking/subgraphs:renderer_cpu", @@ -58,6 +65,39 @@ mediapipe_binary_graph( deps = [":mobile_calculators"], ) +cc_library( + name = "multi_hand_desktop_tflite_calculators", + deps = [ + ":desktop_offline_calculators", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_cpu", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_cpu", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_cpu", + ], +) + +cc_library( + name = "multi_hand_mobile_calculators", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_gpu", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_gpu", + "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_gpu", + ], +) + +mediapipe_binary_graph( + name = "multi_hand_tracking_mobile_gpu_binary_graph", + graph = "multi_hand_tracking_mobile.pbtxt", + output_name = "multi_hand_tracking_mobile_gpu.binarypb", + deps = [":multi_hand_mobile_calculators"], +) + cc_library( name = "detection_mobile_calculators", deps = [ diff --git a/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt index 9e6fdad0..363bb518 100644 --- a/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs hand detection on desktop with TensorFlow Lite # on CPU. # Used in the example in -# mediapipie/examples/desktop/hand_tracking:hand_detection_cpu. +# mediapipe/examples/desktop/hand_tracking:hand_detection_cpu. # Images coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt b/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt index f5431c89..7495c62f 100644 --- a/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs hand detection with TensorFlow Lite on GPU. # Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and -# mediapipie/examples/ios/handdetectiongpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and +# mediapipe/examples/ios/handdetectiongpu. # Images coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt b/mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt index 29ad822a..bd089e77 100644 --- a/mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite # on CPU. # Used in the example in -# mediapipie/examples/desktop/hand_tracking:hand_tracking_tflite. +# mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite. # max_queue_size limits the number of packets enqueued on any input stream # by throttling inputs to the graph. This makes the graph only process one diff --git a/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt b/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt index dc8cb3d1..04cb28db 100644 --- a/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU. # Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and -# mediapipie/examples/ios/handtrackinggpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and +# mediapipe/examples/ios/handtrackinggpu. # Images coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt new file mode 100644 index 00000000..1a554629 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt @@ -0,0 +1,127 @@ +# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow +# Lite on CPU. +# Used in the example in +# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_tflite. + +# max_queue_size limits the number of packets enqueued on any input stream +# by throttling inputs to the graph. This makes the graph only process one +# frame per time. +max_queue_size: 1 + +# Decodes an input video file into images and a video header. +node { + calculator: "OpenCvVideoDecoderCalculator" + input_side_packet: "INPUT_FILE_PATH:input_video_path" + output_stream: "VIDEO:input_video" + output_stream: "VIDEO_PRESTREAM:input_video_header" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided min_size. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks" + output_stream: "prev_has_enough_hands" + node_options: { + [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify max_vec_size in + # ClipVectorSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt + min_size: 2 + } + } +} + +# Drops the incoming image if the previous frame had at least N hands. +# Otherwise, passes the incoming image through to trigger a new round of hand +# detection in MultiHandDetectionSubgraph. +node { + calculator: "GateCalculator" + input_stream: "input_video" + input_stream: "DISALLOW:prev_has_enough_hands" + output_stream: "multi_hand_detection_input_video" + node_options: { + [type.googleapis.com/mediapipe.GateCalculatorOptions] { + empty_packets_as_allow: true + } + } +} + +# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt). +node { + calculator: "MultiHandDetectionSubgraph" + input_stream: "multi_hand_detection_input_video" + output_stream: "DETECTIONS:multi_palm_detections" + output_stream: "NORM_RECTS:multi_palm_rects" +} + +# Subgraph that localizes hand landmarks for multiple hands (see +# multi_hand_landmark.pbtxt). +node { + calculator: "MultiHandLandmarkSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks" +} + +# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the +# arrival of the next input image sends out the cached rectangle with the +# timestamp replaced by that of the input image, essentially generating a packet +# that carries the previous hand rectangle. Note that upon the arrival of the +# very first input image, an empty packet is sent out to jump start the +# feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:input_video" + input_stream: "LOOP:multi_hand_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks" +} + +# Performs association between NormalizedRect vector elements from previous +# frame and those from the current frame if MultiHandDetectionSubgraph runs. +# This calculator ensures that the output multi_hand_rects vector doesn't +# contain overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "prev_multi_hand_rects_from_landmarks" + input_stream: "multi_palm_rects" + output_stream: "multi_hand_rects" + node_options: { + [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { + min_similarity_threshold: 0.1 + } + } +} + +# Subgraph that renders annotations and overlays them on top of the input +# images (see multi_hand_renderer_cpu.pbtxt). +node { + calculator: "MultiHandRendererSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "DETECTIONS:multi_palm_detections" + input_stream: "LANDMARKS:multi_hand_landmarks" + input_stream: "NORM_RECTS:0:multi_palm_rects" + input_stream: "NORM_RECTS:1:multi_hand_rects" + output_stream: "IMAGE:output_video" +} + +# Encodes the annotated images into a video file, adopting properties specified +# in the input video header, e.g., video framerate. +node { + calculator: "OpenCvVideoEncoderCalculator" + input_stream: "VIDEO:output_video" + input_stream: "VIDEO_PRESTREAM:input_video_header" + input_side_packet: "OUTPUT_FILE_PATH:output_video_path" + node_options: { + [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { + codec: "avc1" + video_format: "mp4" + } + } +} diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt new file mode 100644 index 00000000..6e4a0331 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt @@ -0,0 +1,103 @@ +# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow +# Lite on CPU. +# Used in the example in +# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_cpu. + +# Images coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided min_size. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks" + output_stream: "prev_has_enough_hands" + node_options: { + [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify max_vec_size in + # ClipVectorSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt + min_size: 2 + } + } +} + +# Drops the incoming image if the previous frame had at least N hands. +# Otherwise, passes the incoming image through to trigger a new round of hand +# detection in MultiHandDetectionSubgraph. +node { + calculator: "GateCalculator" + input_stream: "input_video" + input_stream: "DISALLOW:prev_has_enough_hands" + output_stream: "multi_hand_detection_input_video" + node_options: { + [type.googleapis.com/mediapipe.GateCalculatorOptions] { + empty_packets_as_allow: true + } + } +} + +# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt). +node { + calculator: "MultiHandDetectionSubgraph" + input_stream: "multi_hand_detection_input_video" + output_stream: "DETECTIONS:multi_palm_detections" + output_stream: "NORM_RECTS:multi_palm_rects" +} + +# Subgraph that localizes hand landmarks for multiple hands (see +# multi_hand_landmark.pbtxt). +node { + calculator: "MultiHandLandmarkSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks" +} + +# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the +# arrival of the next input image sends out the cached rectangle with the +# timestamp replaced by that of the input image, essentially generating a packet +# that carries the previous hand rectangle. Note that upon the arrival of the +# very first input image, an empty packet is sent out to jump start the +# feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:input_video" + input_stream: "LOOP:multi_hand_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks" +} + +# Performs association between NormalizedRect vector elements from previous +# frame and those from the current frame if MultiHandDetectionSubgraph runs. +# This calculator ensures that the output multi_hand_rects vector doesn't +# contain overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "prev_multi_hand_rects_from_landmarks" + input_stream: "multi_palm_rects" + output_stream: "multi_hand_rects" + node_options: { + [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { + min_similarity_threshold: 0.1 + } + } +} + +# Subgraph that renders annotations and overlays them on top of the input +# images (see multi_hand_renderer_cpu.pbtxt). +node { + calculator: "MultiHandRendererSubgraph" + input_stream: "IMAGE:input_video" + input_stream: "DETECTIONS:multi_palm_detections" + input_stream: "LANDMARKS:multi_hand_landmarks" + input_stream: "NORM_RECTS:0:multi_palm_rects" + input_stream: "NORM_RECTS:1:multi_hand_rects" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt b/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt new file mode 100644 index 00000000..c47bc3d8 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt @@ -0,0 +1,123 @@ +# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU. +# Used in the examples in +# mediapipe/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu. + +# Images coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:multi_hand_rects" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided min_size. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks" + output_stream: "prev_has_enough_hands" + node_options: { + [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify max_vec_size in + # ClipVectorSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt + min_size: 2 + } + } +} + +# Drops the incoming image if the previous frame had at least N hands. +# Otherwise, passes the incoming image through to trigger a new round of hand +# detection in MultiHandDetectionSubgraph. +node { + calculator: "GateCalculator" + input_stream: "throttled_input_video" + input_stream: "DISALLOW:prev_has_enough_hands" + output_stream: "multi_hand_detection_input_video" + node_options: { + [type.googleapis.com/mediapipe.GateCalculatorOptions] { + empty_packets_as_allow: true + } + } +} + +# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt). +node { + calculator: "MultiHandDetectionSubgraph" + input_stream: "multi_hand_detection_input_video" + output_stream: "DETECTIONS:multi_palm_detections" + output_stream: "NORM_RECTS:multi_palm_rects" +} + +# Subgraph that localizes hand landmarks for multiple hands (see +# multi_hand_landmark.pbtxt). +node { + calculator: "MultiHandLandmarkSubgraph" + input_stream: "IMAGE:throttled_input_video" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "LANDMARKS:multi_hand_landmarks" + output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks" +} + +# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the +# arrival of the next input image sends out the cached rectangle with the +# timestamp replaced by that of the input image, essentially generating a packet +# that carries the previous hand rectangle. Note that upon the arrival of the +# very first input image, an empty packet is sent out to jump start the +# feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:throttled_input_video" + input_stream: "LOOP:multi_hand_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks" +} + +# Performs association between NormalizedRect vector elements from previous +# frame and those from the current frame if MultiHandDetectionSubgraph runs. +# This calculator ensures that the output multi_hand_rects vector doesn't +# contain overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "prev_multi_hand_rects_from_landmarks" + input_stream: "multi_palm_rects" + output_stream: "multi_hand_rects" + node_options: { + [type.googleapis.com/mediapipe.AssociationCalculatorOptions] { + min_similarity_threshold: 0.1 + } + } +} + +# Subgraph that renders annotations and overlays them on top of the input +# images (see multi_hand_renderer_gpu.pbtxt). +node { + calculator: "MultiHandRendererSubgraph" + input_stream: "IMAGE:throttled_input_video" + input_stream: "DETECTIONS:multi_palm_detections" + input_stream: "LANDMARKS:multi_hand_landmarks" + input_stream: "NORM_RECTS:0:multi_palm_rects" + input_stream: "NORM_RECTS:1:multi_hand_rects" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/hand_tracking/subgraphs/BUILD b/mediapipe/graphs/hand_tracking/subgraphs/BUILD index 93a0d104..7692ec77 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/BUILD +++ b/mediapipe/graphs/hand_tracking/subgraphs/BUILD @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -licenses(["notice"]) # Apache 2.0 - -package(default_visibility = ["//visibility:public"]) - load( "//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph", ) +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:public"]) + mediapipe_simple_subgraph( name = "hand_detection_cpu", graph = "hand_detection_cpu.pbtxt", @@ -42,6 +42,29 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "multi_hand_detection_cpu", + graph = "multi_hand_detection_cpu.pbtxt", + register_as = "MultiHandDetectionSubgraph", + deps = [ + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/image:image_transformation_calculator", + "//mediapipe/calculators/tflite:ssd_anchors_calculator", + "//mediapipe/calculators/tflite:tflite_converter_calculator", + "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator", + "//mediapipe/calculators/tflite:tflite_inference_calculator", + "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator", + "//mediapipe/calculators/util:detection_label_id_to_text_calculator", + "//mediapipe/calculators/util:detection_letterbox_removal_calculator", + "//mediapipe/calculators/util:detections_to_rects_calculator", + "//mediapipe/calculators/util:non_max_suppression_calculator", + "//mediapipe/calculators/util:rect_transformation_calculator", + ], +) + mediapipe_simple_subgraph( name = "hand_landmark_cpu", graph = "hand_landmark_cpu.pbtxt", @@ -65,6 +88,18 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "multi_hand_landmark_cpu", + graph = "multi_hand_landmark.pbtxt", + register_as = "MultiHandLandmarkSubgraph", + deps = [ + ":hand_landmark_cpu", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/util:filter_collection_calculator", + ], +) + mediapipe_simple_subgraph( name = "renderer_cpu", graph = "renderer_cpu.pbtxt", @@ -77,6 +112,20 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "multi_hand_renderer_cpu", + graph = "multi_hand_renderer_cpu.pbtxt", + register_as = "MultiHandRendererSubgraph", + deps = [ + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/calculators/util:rect_to_render_data_calculator", + ], +) + mediapipe_simple_subgraph( name = "hand_detection_gpu", graph = "hand_detection_gpu.pbtxt", @@ -97,6 +146,29 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "multi_hand_detection_gpu", + graph = "multi_hand_detection_gpu.pbtxt", + register_as = "MultiHandDetectionSubgraph", + deps = [ + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/image:image_transformation_calculator", + "//mediapipe/calculators/tflite:ssd_anchors_calculator", + "//mediapipe/calculators/tflite:tflite_converter_calculator", + "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator", + "//mediapipe/calculators/tflite:tflite_inference_calculator", + "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator", + "//mediapipe/calculators/util:detection_label_id_to_text_calculator", + "//mediapipe/calculators/util:detection_letterbox_removal_calculator", + "//mediapipe/calculators/util:detections_to_rects_calculator", + "//mediapipe/calculators/util:non_max_suppression_calculator", + "//mediapipe/calculators/util:rect_transformation_calculator", + ], +) + mediapipe_simple_subgraph( name = "hand_landmark_gpu", graph = "hand_landmark_gpu.pbtxt", @@ -119,6 +191,18 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "multi_hand_landmark_gpu", + graph = "multi_hand_landmark.pbtxt", + register_as = "MultiHandLandmarkSubgraph", + deps = [ + ":hand_landmark_gpu", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/util:filter_collection_calculator", + ], +) + mediapipe_simple_subgraph( name = "renderer_gpu", graph = "renderer_gpu.pbtxt", @@ -130,3 +214,17 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:rect_to_render_data_calculator", ], ) + +mediapipe_simple_subgraph( + name = "multi_hand_renderer_gpu", + graph = "multi_hand_renderer_gpu.pbtxt", + register_as = "MultiHandRendererSubgraph", + deps = [ + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/calculators/util:rect_to_render_data_calculator", + ], +) diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt new file mode 100644 index 00000000..aa055731 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt @@ -0,0 +1,213 @@ +# MediaPipe multi-hand detection subgraph. + +type: "MultiHandDetectionSubgraph" + +input_stream: "input_video" +output_stream: "DETECTIONS:palm_detections" +output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections" + +# Transforms the input image on CPU to a 256x256 image. To scale the input +# image, the scale_mode option is set to FIT to preserve the aspect ratio, +# resulting in potential letterboxing in the transformed image. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE:input_video" + output_stream: "IMAGE:transformed_input_video" + output_stream: "LETTERBOX_PADDING:letterbox_padding" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + output_width: 256 + output_height: 256 + scale_mode: FIT + } + } +} + +# Generates a single side packet containing a TensorFlow Lite op resolver that +# supports custom ops needed by the model used in this graph. +node { + calculator: "TfLiteCustomOpResolverCalculator" + output_side_packet: "opresolver" +} + +# Converts the transformed input image on CPU into an image tensor as a +# TfLiteTensor. The zero_center option is set to true to normalize the +# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. +node { + calculator: "TfLiteConverterCalculator" + input_stream: "IMAGE:transformed_input_video" + output_stream: "TENSORS:image_tensor" +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS:image_tensor" + output_stream: "TENSORS:detection_tensors" + input_side_packet: "CUSTOM_OP_RESOLVER:opresolver" + node_options: { + [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { + model_path: "mediapipe/models/palm_detection.tflite" + } + } +} + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + node_options: { + [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { + num_layers: 5 + min_scale: 0.1171875 + max_scale: 0.75 + input_size_height: 256 + input_size_width: 256 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 8 + strides: 16 + strides: 32 + strides: 32 + strides: 32 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TfLiteTensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { + num_classes: 1 + num_boxes: 2944 + num_coords: 18 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 7 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + + x_scale: 256.0 + y_scale: 256.0 + h_scale: 256.0 + w_scale: 256.0 + min_score_thresh: 0.5 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + node_options: { + [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { + min_suppression_threshold: 0.3 + min_score_threshold: 0.5 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + return_empty_detections: true + } + } +} + +# Maps detection label IDs to the corresponding label text ("Palm"). The label +# map is provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "labeled_detections" + node_options: { + [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { + label_map_path: "mediapipe/models/palm_detection_labelmap.txt" + } + } +} + +# Adjusts detection locations (already normalized to [0.f, 1.f]) on the +# letterboxed image (after image transformation with the FIT scale mode) to the +# corresponding locations on the same image with the letterbox removed (the +# input image to the graph before image transformation). +node { + calculator: "DetectionLetterboxRemovalCalculator" + input_stream: "DETECTIONS:labeled_detections" + input_stream: "LETTERBOX_PADDING:letterbox_padding" + output_stream: "DETECTIONS:palm_detections" +} + +# Extracts image size from the input images. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:input_video" + output_stream: "SIZE:image_size" +} + +# Converts each palm detection into a rectangle (normalized by image size) +# that encloses the palm and is rotated such that the line connecting center of +# the wrist and MCP of the middle finger is aligned with the Y-axis of the +# rectangle. +node { + calculator: "DetectionsToRectsCalculator" + input_stream: "DETECTIONS:palm_detections" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "NORM_RECTS:palm_rects" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] { + rotation_vector_start_keypoint_index: 0 # Center of wrist. + rotation_vector_end_keypoint_index: 2 # MCP of middle finger. + rotation_vector_target_angle_degrees: 90 + output_zero_rect_for_empty_detections: true + } + } +} + +# Expands and shifts the rectangle that contains the palm so that it's likely +# to cover the entire hand. +node { + calculator: "RectTransformationCalculator" + input_stream: "NORM_RECTS:palm_rects" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] { + scale_x: 2.6 + scale_y: 2.6 + shift_y: -0.5 + square_long: true + } + } +} + +# Clips the size of the input vector to the provided max_vec_size. This +# determines the maximum number of hand instances this graph outputs. +# Note that the performance gain of clipping detections earlier in this graph is +# minimal because NMS will minimize overlapping detections and the number of +# detections isn't expected to exceed 5-10. +node { + calculator: "ClipNormalizedRectVectorSizeCalculator" + input_stream: "hand_rects_from_palm_detections" + output_stream: "clipped_hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify min_size in + # CollectionHsMinSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt. + max_vec_size: 2 + } + } +} diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt new file mode 100644 index 00000000..afd1fd15 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt @@ -0,0 +1,218 @@ +# MediaPipe multi-hand detection subgraph. + +type: "MultiHandDetectionSubgraph" + +input_stream: "input_video" +output_stream: "DETECTIONS:palm_detections" +output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections" + +# Transforms the input image on GPU to a 256x256 image. To scale the input +# image, the scale_mode option is set to FIT to preserve the aspect ratio, +# resulting in potential letterboxing in the transformed image. +node: { + calculator: "ImageTransformationCalculator" + input_stream: "IMAGE_GPU:input_video" + output_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "LETTERBOX_PADDING:letterbox_padding" + node_options: { + [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { + output_width: 256 + output_height: 256 + scale_mode: FIT + } + } +} + +# Generates a single side packet containing a TensorFlow Lite op resolver that +# supports custom ops needed by the model used in this graph. +node { + calculator: "TfLiteCustomOpResolverCalculator" + output_side_packet: "opresolver" + node_options: { + [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] { + use_gpu: true + } + } +} + +# Converts the transformed input image on GPU into an image tensor stored as a +# TfLiteTensor. +node { + calculator: "TfLiteConverterCalculator" + input_stream: "IMAGE_GPU:transformed_input_video" + output_stream: "TENSORS_GPU:image_tensor" +} + +# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS_GPU:image_tensor" + output_stream: "TENSORS_GPU:detection_tensors" + input_side_packet: "CUSTOM_OP_RESOLVER:opresolver" + node_options: { + [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { + model_path: "mediapipe/models/palm_detection.tflite" + use_gpu: true + } + } +} + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + node_options: { + [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] { + num_layers: 5 + min_scale: 0.1171875 + max_scale: 0.75 + input_size_height: 256 + input_size_width: 256 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + strides: 8 + strides: 16 + strides: 32 + strides: 32 + strides: 32 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TfLiteTensorsToDetectionsCalculator" + input_stream: "TENSORS_GPU:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] { + num_classes: 1 + num_boxes: 2944 + num_coords: 18 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 7 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + + x_scale: 256.0 + y_scale: 256.0 + h_scale: 256.0 + w_scale: 256.0 + min_score_thresh: 0.7 + } + } +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "detections" + output_stream: "filtered_detections" + node_options: { + [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + return_empty_detections: true + } + } +} + +# Maps detection label IDs to the corresponding label text ("Palm"). The label +# map is provided in the label_map_path option. +node { + calculator: "DetectionLabelIdToTextCalculator" + input_stream: "filtered_detections" + output_stream: "labeled_detections" + node_options: { + [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { + label_map_path: "mediapipe/models/palm_detection_labelmap.txt" + } + } +} + +# Adjusts detection locations (already normalized to [0.f, 1.f]) on the +# letterboxed image (after image transformation with the FIT scale mode) to the +# corresponding locations on the same image with the letterbox removed (the +# input image to the graph before image transformation). +node { + calculator: "DetectionLetterboxRemovalCalculator" + input_stream: "DETECTIONS:labeled_detections" + input_stream: "LETTERBOX_PADDING:letterbox_padding" + output_stream: "DETECTIONS:palm_detections" +} + +# Extracts image size from the input images. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:input_video" + output_stream: "SIZE:image_size" +} + +# Converts each palm detection into a rectangle (normalized by image size) +# that encloses the palm and is rotated such that the line connecting center of +# the wrist and MCP of the middle finger is aligned with the Y-axis of the +# rectangle. +node { + calculator: "DetectionsToRectsCalculator" + input_stream: "DETECTIONS:palm_detections" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "NORM_RECTS:palm_rects" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] { + rotation_vector_start_keypoint_index: 0 # Center of wrist. + rotation_vector_end_keypoint_index: 2 # MCP of middle finger. + rotation_vector_target_angle_degrees: 90 + output_zero_rect_for_empty_detections: true + } + } +} + +# Expands and shifts the rectangle that contains the palm so that it's likely +# to cover the entire hand. +node { + calculator: "RectTransformationCalculator" + input_stream: "NORM_RECTS:palm_rects" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] { + scale_x: 2.6 + scale_y: 2.6 + shift_y: -0.5 + square_long: true + } + } +} + +# Clips the size of the input vector to the provided max_vec_size. This +# determines the maximum number of hand instances this graph outputs. +# Note that the performance gain of clipping detections earlier in this graph is +# minimal because NMS will minimize overlapping detections and the number of +# detections isn't expected to exceed 5-10. +node { + calculator: "ClipNormalizedRectVectorSizeCalculator" + input_stream: "hand_rects_from_palm_detections" + output_stream: "clipped_hand_rects_from_palm_detections" + node_options: { + [type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] { + # This value can be changed to support tracking arbitrary number of hands. + # Please also remember to modify min_size in + # CollectionHsMinSizeCalculatorOptions in + # mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt and + # mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt. + max_vec_size: 2 + } + } +} diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt new file mode 100644 index 00000000..a380966c --- /dev/null +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt @@ -0,0 +1,84 @@ +# MediaPipe hand landmark localization subgraph. + +type: "MultiHandLandmarkSubgraph" + +input_stream: "IMAGE:input_video" +# A vector of NormalizedRect, one per each hand detected. +input_stream: "NORM_RECTS:multi_hand_rects" +# A vector of NormalizedLandmarks, one set per each hand. +output_stream: "LANDMARKS:filtered_multi_hand_landmarks" +# A vector of NormalizedRect, one per each hand. +output_stream: "NORM_RECTS:filtered_multi_hand_rects_for_next_frame" + +# Outputs each element of multi_hand_rects at a fake timestamp for the rest +# of the graph to process. Clones the input_video packet for each +# single_hand_rect at the fake timestamp. At the end of the loop, +# outputs the BATCH_END timestamp for downstream calculators to inform them +# that all elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:multi_hand_rects" + input_stream: "CLONE:input_video" + output_stream: "ITEM:single_hand_rect" + output_stream: "CLONE:input_video_cloned" + output_stream: "BATCH_END:single_hand_rect_timestamp" +} + +node { + calculator: "HandLandmarkSubgraph" + input_stream: "IMAGE:input_video_cloned" + input_stream: "NORM_RECT:single_hand_rect" + output_stream: "LANDMARKS:single_hand_landmarks" + output_stream: "NORM_RECT:single_hand_rect_from_landmarks" + output_stream: "PRESENCE:single_hand_presence" +} + +# Collects the boolean presence value for each single hand into a vector. Upon +# receiving the BATCH_END timestamp, outputs a vector of boolean values at the +# BATCH_END timestamp. +node { + calculator: "EndLoopBooleanCalculator" + input_stream: "ITEM:single_hand_presence" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_presence" +} + +# Collects a set of landmarks for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarksVectorCalculator" + input_stream: "ITEM:single_hand_landmarks" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_landmarks" +} + +# Collects a NormalizedRect for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:single_hand_rect_from_landmarks" + input_stream: "BATCH_END:single_hand_rect_timestamp" + output_stream: "ITERABLE:multi_hand_rects_for_next_frame" +} + +# Filters the input vector of landmarks based on hand presence value for each +# hand. If the hand presence for hand #i is false, the set of landmarks +# corresponding to that hand are dropped from the vector. +node { + calculator: "FilterLandmarksCollectionCalculator" + input_stream: "ITERABLE:multi_hand_landmarks" + input_stream: "CONDITION:multi_hand_presence" + output_stream: "ITERABLE:filtered_multi_hand_landmarks" +} + +# Filters the input vector of NormalizedRect based on hand presence value for +# each hand. If the hand presence for hand #i is false, the NormalizedRect +# corresponding to that hand are dropped from the vector. +node { + calculator: "FilterNormalizedRectCollectionCalculator" + input_stream: "ITERABLE:multi_hand_rects_for_next_frame" + input_stream: "CONDITION:multi_hand_presence" + output_stream: "ITERABLE:filtered_multi_hand_rects_for_next_frame" +} diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt new file mode 100644 index 00000000..2dcd6b47 --- /dev/null +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt @@ -0,0 +1,144 @@ +# MediaPipe multi-hand tracking rendering subgraph. + +type: "MultiHandRendererSubgraph" + +input_stream: "IMAGE:input_image" +# A vector of NormalizedLandmarks, one for each hand. +input_stream: "LANDMARKS:multi_hand_landmarks" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:0:multi_palm_rects" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:1:multi_hand_rects" +# A vector of Detection, one for each hand. +input_stream: "DETECTIONS:palm_detections" +output_stream: "IMAGE:output_image" + +# Converts detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:palm_detections" + output_stream: "RENDER_DATA:detection_render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 0 g: 255 b: 0 } + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "RENDER_DATA:multi_hand_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 255 g: 0 b: 0 } + thickness: 4.0 + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_palm_rects" + output_stream: "RENDER_DATA:multi_palm_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 125 g: 0 b: 122 } + thickness: 4.0 + } + } +} + +# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest +# of the graph to process. At the end of the loop, outputs the BATCH_END +# timestamp for downstream calculators to inform them that all elements in the +# vector have been processed. +node { + calculator: "BeginLoopNormalizedLandmarksVectorCalculator" + input_stream: "ITERABLE:multi_hand_landmarks" + output_stream: "ITEM:single_hand_landmarks" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to drawing primitives for annotation overlay. +node { + calculator: "LandmarksToRenderDataCalculator" + input_stream: "NORM_LANDMARKS:single_hand_landmarks" + output_stream: "RENDER_DATA:single_hand_landmark_render_data" + node_options: { + [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] { + landmark_connections: 0 + landmark_connections: 1 + landmark_connections: 1 + landmark_connections: 2 + landmark_connections: 2 + landmark_connections: 3 + landmark_connections: 3 + landmark_connections: 4 + landmark_connections: 0 + landmark_connections: 5 + landmark_connections: 5 + landmark_connections: 6 + landmark_connections: 6 + landmark_connections: 7 + landmark_connections: 7 + landmark_connections: 8 + landmark_connections: 5 + landmark_connections: 9 + landmark_connections: 9 + landmark_connections: 10 + landmark_connections: 10 + landmark_connections: 11 + landmark_connections: 11 + landmark_connections: 12 + landmark_connections: 9 + landmark_connections: 13 + landmark_connections: 13 + landmark_connections: 14 + landmark_connections: 14 + landmark_connections: 15 + landmark_connections: 15 + landmark_connections: 16 + landmark_connections: 13 + landmark_connections: 17 + landmark_connections: 0 + landmark_connections: 17 + landmark_connections: 17 + landmark_connections: 18 + landmark_connections: 18 + landmark_connections: 19 + landmark_connections: 19 + landmark_connections: 20 + landmark_color { r: 255 g: 0 b: 0 } + connection_color { r: 0 g: 255 b: 0 } + thickness: 4.0 + } + } +} + +# Collects a RenderData object for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END +# timestamp. +node { + calculator: "EndLoopRenderDataCalculator" + input_stream: "ITEM:single_hand_landmark_render_data" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_hand_landmarks_render_data" +} + +# Draws annotations and overlays them on top of the input images. Consumes +# a vector of RenderData objects and draws each of them on the input frame. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "INPUT_FRAME:input_image" + input_stream: "detection_render_data" + input_stream: "multi_hand_rects_render_data" + input_stream: "multi_palm_rects_render_data" + input_stream: "VECTOR:0:multi_hand_landmarks_render_data" + output_stream: "OUTPUT_FRAME:output_image" +} diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt new file mode 100644 index 00000000..3ea9275d --- /dev/null +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt @@ -0,0 +1,144 @@ +# MediaPipe multi-hand tracking rendering subgraph. + +type: "MultiHandRendererSubgraph" + +input_stream: "IMAGE:input_image" +# A vector of NormalizedLandmarks, one for each hand. +input_stream: "LANDMARKS:multi_hand_landmarks" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:0:multi_palm_rects" +# A vector of NormalizedRect, one for each hand. +input_stream: "NORM_RECTS:1:multi_hand_rects" +# A vector of Detection, one for each hand. +input_stream: "DETECTIONS:palm_detections" +output_stream: "IMAGE:output_image" + +# Converts detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:palm_detections" + output_stream: "RENDER_DATA:detection_render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 0 g: 255 b: 0 } + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_hand_rects" + output_stream: "RENDER_DATA:multi_hand_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 255 g: 0 b: 0 } + thickness: 4.0 + } + } +} + +# Converts normalized rects to drawing primitives for annotation overlay. +node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECTS:multi_palm_rects" + output_stream: "RENDER_DATA:multi_palm_rects_render_data" + node_options: { + [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { + filled: false + color { r: 125 g: 0 b: 122 } + thickness: 4.0 + } + } +} + +# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest +# of the graph to process. At the end of the loop, outputs the BATCH_END +# timestamp for downstream calculators to inform them that all elements in the +# vector have been processed. +node { + calculator: "BeginLoopNormalizedLandmarksVectorCalculator" + input_stream: "ITERABLE:multi_hand_landmarks" + output_stream: "ITEM:single_hand_landmarks" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to drawing primitives for annotation overlay. +node { + calculator: "LandmarksToRenderDataCalculator" + input_stream: "NORM_LANDMARKS:single_hand_landmarks" + output_stream: "RENDER_DATA:single_hand_landmark_render_data" + node_options: { + [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] { + landmark_connections: 0 + landmark_connections: 1 + landmark_connections: 1 + landmark_connections: 2 + landmark_connections: 2 + landmark_connections: 3 + landmark_connections: 3 + landmark_connections: 4 + landmark_connections: 0 + landmark_connections: 5 + landmark_connections: 5 + landmark_connections: 6 + landmark_connections: 6 + landmark_connections: 7 + landmark_connections: 7 + landmark_connections: 8 + landmark_connections: 5 + landmark_connections: 9 + landmark_connections: 9 + landmark_connections: 10 + landmark_connections: 10 + landmark_connections: 11 + landmark_connections: 11 + landmark_connections: 12 + landmark_connections: 9 + landmark_connections: 13 + landmark_connections: 13 + landmark_connections: 14 + landmark_connections: 14 + landmark_connections: 15 + landmark_connections: 15 + landmark_connections: 16 + landmark_connections: 13 + landmark_connections: 17 + landmark_connections: 0 + landmark_connections: 17 + landmark_connections: 17 + landmark_connections: 18 + landmark_connections: 18 + landmark_connections: 19 + landmark_connections: 19 + landmark_connections: 20 + landmark_color { r: 255 g: 0 b: 0 } + connection_color { r: 0 g: 255 b: 0 } + thickness: 4.0 + } + } +} + +# Collects a RenderData object for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END +# timestamp. +node { + calculator: "EndLoopRenderDataCalculator" + input_stream: "ITEM:single_hand_landmark_render_data" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_hand_landmarks_render_data" +} + +# Draws annotations and overlays them on top of the input images. Consumes +# a vector of RenderData objects and draws each of them on the input frame. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "detection_render_data" + input_stream: "multi_hand_rects_render_data" + input_stream: "multi_palm_rects_render_data" + input_stream: "VECTOR:0:multi_hand_landmarks_render_data" + output_stream: "OUTPUT_FRAME_GPU:output_image" +} diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt index 899785a1..7cf19b2e 100644 --- a/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt @@ -1,6 +1,6 @@ # MediaPipe graph that performs object detection with TensorFlow Lite on CPU. # Used in the examples in -# mediapipie/examples/desktop/object_detection:object_detection_cpu. +# mediapipe/examples/desktop/object_detection:object_detection_cpu. # Images on CPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt index 575d933a..fd63fd97 100644 --- a/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs object detection on desktop with TensorFlow Lite # on CPU. # Used in the example in -# mediapipie/examples/desktop/object_detection:object_detection_tflite. +# mediapipe/examples/desktop/object_detection:object_detection_tflite. # max_queue_size limits the number of packets enqueued on any input stream # by throttling inputs to the graph. This makes the graph only process one diff --git a/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt b/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt index 3e0e4e6d..4de82c07 100644 --- a/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs object detection with TensorFlow Lite on CPU. # Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/objectdetectioncpu and -# mediapipie/examples/ios/objectdetectioncpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/objectdetectioncpu and +# mediapipe/examples/ios/objectdetectioncpu. # Images on GPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt b/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt index dfed1669..f3dc1d9e 100644 --- a/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt @@ -1,7 +1,7 @@ # MediaPipe graph that performs object detection with TensorFlow Lite on GPU. # Used in the examples in -# mediapipie/examples/android/src/java/com/mediapipe/apps/objectdetectiongpu and -# mediapipie/examples/ios/objectdetectiongpu. +# mediapipe/examples/android/src/java/com/mediapipe/apps/objectdetectiongpu and +# mediapipe/examples/ios/objectdetectiongpu. # Images on GPU coming into and out of the graph. input_stream: "input_video" diff --git a/mediapipe/util/sequence/BUILD b/mediapipe/util/sequence/BUILD index 82f9e8c9..1e07d0ca 100644 --- a/mediapipe/util/sequence/BUILD +++ b/mediapipe/util/sequence/BUILD @@ -28,7 +28,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -50,7 +50,7 @@ cc_library( "//mediapipe/framework/port:status", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -61,7 +61,7 @@ cc_test( ":media_sequence_util", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -74,6 +74,6 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) diff --git a/third_party/opencv_linux.BUILD b/third_party/opencv_linux.BUILD index 3e57c5c8..ef2aeb49 100644 --- a/third_party/opencv_linux.BUILD +++ b/third_party/opencv_linux.BUILD @@ -14,6 +14,8 @@ cc_library( srcs = glob( [ "lib/x86_64-linux-gnu/libopencv_core.so", + "lib/x86_64-linux-gnu/libopencv_calib3d.so", + "lib/x86_64-linux-gnu/libopencv_features2d.so", "lib/x86_64-linux-gnu/libopencv_highgui.so", "lib/x86_64-linux-gnu/libopencv_imgcodecs.so", "lib/x86_64-linux-gnu/libopencv_imgproc.so", diff --git a/third_party/opencv_macos.BUILD b/third_party/opencv_macos.BUILD index 275f31ec..be1733e0 100644 --- a/third_party/opencv_macos.BUILD +++ b/third_party/opencv_macos.BUILD @@ -13,6 +13,8 @@ cc_library( srcs = glob( [ "local/opt/opencv@3/lib/libopencv_core.dylib", + "local/opt/opencv@3/lib/libopencv_calib3d.dylib", + "local/opt/opencv@3/lib/libopencv_features2d.dylib", "local/opt/opencv@3/lib/libopencv_highgui.dylib", "local/opt/opencv@3/lib/libopencv_imgcodecs.dylib", "local/opt/opencv@3/lib/libopencv_imgproc.dylib", diff --git a/third_party/rules_apple_c0863d0596ae6b769a29fa3fb72ff036444fd249.diff b/third_party/rules_apple_c0863d0596ae6b769a29fa3fb72ff036444fd249.diff deleted file mode 100644 index 736292cf..00000000 --- a/third_party/rules_apple_c0863d0596ae6b769a29fa3fb72ff036444fd249.diff +++ /dev/null @@ -1,25 +0,0 @@ -commit c0863d0596ae6b769a29fa3fb72ff036444fd249 (HEAD -> py3) -Author: Camillo Lugaresi -Date: Fri Aug 16 00:13:16 2019 -0700 - - Fix codesigningtool.py py3 compatibility. - - In recent versions of plistlib, binary data entries are returned as instances of the built-in bytes class, and plistlib.Data is deprecated. - Since this script was expecting a plistlib.Data, it would fail with the error "AttributeError: 'bytes' object has no attribute 'data'". - This change makes it compatible with both new and old versions of plistlib. - -diff --git a/tools/codesigningtool/codesigningtool.py b/tools/codesigningtool/codesigningtool.py -index 59f3841..40cdcf3 100644 ---- a/tools/codesigningtool/codesigningtool.py -+++ b/tools/codesigningtool/codesigningtool.py -@@ -102,7 +102,9 @@ def _certificate_fingerprint(identity): - def _get_identities_from_provisioning_profile(mpf): - """Iterates through all the identities in a provisioning profile, lazily.""" - for identity in mpf["DeveloperCertificates"]: -- yield _certificate_fingerprint(identity.data) -+ if not _PY3: -+ identity = identity.data -+ yield _certificate_fingerprint(identity) - - - def _find_codesign_identities(identity=None): diff --git a/third_party/tensorflow_065c20bf79253257c87bd4614bb9a7fdef015cbb.diff b/third_party/tensorflow_065c20bf79253257c87bd4614bb9a7fdef015cbb.diff deleted file mode 100644 index 16f9c326..00000000 --- a/third_party/tensorflow_065c20bf79253257c87bd4614bb9a7fdef015cbb.diff +++ /dev/null @@ -1,22 +0,0 @@ -commit 065c20bf79253257c87bd4614bb9a7fdef015cbb -Author: Camillo Lugaresi -Date: Thu Aug 15 18:34:41 2019 -0700 - - Use python3 if available to run gen_git_source.py. - - gen_git_source.py fails with an "ImportError: No module named builtins" on a default installation of Python 2 (at least, the one that comes with macOS). This can be worked around by installing the "future" package from pip. However, instead of requiring users to go through this extra step, we can simply run the script using Python 3 if it's installed. The script works on a default installation of Python 3, without requiring extra packages. - -diff --git a/third_party/git/git_configure.bzl b/third_party/git/git_configure.bzl -index fc18fdb988..3ce64242af 100644 ---- a/third_party/git/git_configure.bzl -+++ b/third_party/git/git_configure.bzl -@@ -18,6 +18,9 @@ def _get_python_bin(repository_ctx): - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin -+ python_bin_path = repository_ctx.which("python3") -+ if python_bin_path != None: -+ return str(python_bin_path) - python_bin_path = repository_ctx.which("python") - if python_bin_path != None: - return str(python_bin_path) diff --git a/third_party/tensorflow_f67fcbefce906cd419e4657f0d41e21019b71abd.diff b/third_party/tensorflow_f67fcbefce906cd419e4657f0d41e21019b71abd.diff deleted file mode 100644 index 080e4dc1..00000000 --- a/third_party/tensorflow_f67fcbefce906cd419e4657f0d41e21019b71abd.diff +++ /dev/null @@ -1,24 +0,0 @@ -commit f67fcbefce906cd419e4657f0d41e21019b71abd (HEAD -> formediapipe) -Author: Camillo Lugaresi -Date: Fri Aug 16 12:24:58 2019 -0700 - - elementwise requires C++14 - - This file fails to compile when using C++11, which is the default. This can be worked around by passing --cxxopt='-std=c++14' as a global build option to Bazel, but it is more convenient for users if we just configure this cc_library to be built with C++14 by default. - - The authors may also want to change it to be compatible with C++11, but that's out of scope for this change. - -diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD -index 17e59e70eb..4302a1f644 100644 ---- a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD -+++ b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD -@@ -197,6 +197,9 @@ cc_library( - name = "elementwise", - srcs = ["elementwise.cc"], - hdrs = ["elementwise.h"], -+ copts = [ -+ "-std=c++14", -+ ], - deps = [ - "//tensorflow/lite/delegates/gpu/common:model", - "//tensorflow/lite/delegates/gpu/common:operations",