Project import generated by Copybara.

GitOrigin-RevId: 87e46800807001e01d686fd7bcc2533714556920

.bazelrc

@@ -19,6 +19,9 @@ build --incompatible_depset_is_not_iterable=false
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
 
+# Allow debugging with XCODE
+build --apple_generate_dsym
+
 # Android configs.
 build:android --crosstool_top=//external:android/crosstool
 build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain

Dockerfile

@@ -30,10 +30,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         unzip \
         python \
         python-pip \
+        python3-pip \
         libopencv-core-dev \
         libopencv-highgui-dev \
         libopencv-imgproc-dev \
         libopencv-video-dev \
+        libopencv-calib3d-dev \
+        libopencv-features2d-dev \
         software-properties-common && \
     add-apt-repository -y ppa:openjdk-r/ppa && \
     apt-get update && apt-get install -y openjdk-8-jdk && \
@@ -42,9 +45,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN pip install --upgrade setuptools
 RUN pip install future
+RUN pip3 install six
 
 # Install bazel
-ARG BAZEL_VERSION=0.26.1
+ARG BAZEL_VERSION=1.1.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

README.md

@@ -14,12 +14,13 @@
 * [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
 * [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
+* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
 
 ![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
 ![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
-![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
+![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
 
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).

WORKSPACE

@@ -12,17 +12,21 @@ http_archive(
 load("@bazel_skylib//lib:versions.bzl", "versions")
 versions.check(minimum_bazel_version = "0.24.1")
 
-# ABSL cpp library.
+# ABSL cpp library lts_2019_08_08.
 http_archive(
     name = "com_google_absl",
-    # Head commit on 2019-04-12.
-    # TODO: Switch to the latest absl version when the problem gets
-    # fixed.
     urls = [
-        "https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
     ],
-    sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
-    strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
+    # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
+    patches = [
+        "@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+    strip_prefix = "abseil-cpp-20190808",
+    sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
 )
 
 http_archive(
@@ -103,15 +107,22 @@ http_archive(
     ],
 )
 
-# 2019-11-12
-_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
-_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
+# 2019-11-21
+_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
+_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
 http_archive(
     name = "org_tensorflow",
     urls = [
       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
+    # Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
+    patches = [
+        "@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
     sha256 = _TENSORFLOW_SHA256,
 )
@@ -119,8 +130,22 @@ http_archive(
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
 tf_workspace(tf_repo_name = "org_tensorflow")
 
+http_archive(
+    name = "ceres_solver",
+    url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
+    patches = [
+        "@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+    strip_prefix = "ceres-solver-1.14.0",
+    sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
+)
+
 # Please run
 # $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+#                        libopencv-calib3d-dev libopencv-features2d-dev \
 #                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
@@ -288,3 +313,4 @@ http_archive(
     strip_prefix = "google-toolbox-for-mac-2.2.1",
     build_file = "@//third_party:google_toolbox_for_mac.BUILD",
 )
+

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -9,6 +9,7 @@
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
     "mediapipe/examples/ios/handdetectiongpu/BUILD",
     "mediapipe/examples/ios/handtrackinggpu/BUILD",
+    "mediapipe/examples/ios/multihandtrackinggpu/BUILD",
     "mediapipe/examples/ios/objectdetectioncpu/BUILD",
     "mediapipe/examples/ios/objectdetectiongpu/BUILD"
   ],
@@ -18,6 +19,7 @@
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
     "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
     "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
+    "//mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp",
     "//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
     "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
     "//mediapipe/objc:mediapipe_framework_ios"
@@ -84,6 +86,8 @@
     "mediapipe/examples/ios/handdetectiongpu/Base.lproj",
     "mediapipe/examples/ios/handtrackinggpu",
     "mediapipe/examples/ios/handtrackinggpu/Base.lproj",
+    "mediapipe/examples/ios/multihandtrackinggpu",
+    "mediapipe/examples/ios/multihandtrackinggpu/Base.lproj",
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectioncpu/Base.lproj",
     "mediapipe/examples/ios/objectdetectiongpu",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -16,6 +16,7 @@
     "mediapipe/examples/ios/facedetectiongpu",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",
+    "mediapipe/examples/ios/multihandtrackinggpu",
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectiongpu"
   ],

mediapipe/calculators/core/BUILD

@@ -165,6 +165,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
 )
@@ -691,6 +692,7 @@ cc_library(
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",

mediapipe/calculators/core/add_header_calculator.cc

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/logging.h"
 
 namespace mediapipe {
 
-// Attach the header from one stream to another stream.
+// Attach the header from a stream or side input to another stream.
 //
 // The header stream (tag HEADER) must not have any packets in it.
 //
@@ -25,17 +26,53 @@ namespace mediapipe {
 // calculator to not need a header or to accept a separate stream with
 // a header, that would be more future proof.
 //
+// Example usage 1:
+// node {
+//   calculator: "AddHeaderCalculator"
+//   input_stream: "DATA:audio"
+//   input_stream: "HEADER:audio_header"
+//   output_stream: "audio_with_header"
+// }
+//
+// Example usage 2:
+// node {
+//   calculator: "AddHeaderCalculator"
+//   input_stream: "DATA:audio"
+//   input_side_packet: "HEADER:audio_header"
+//   output_stream: "audio_with_header"
+// }
+//
 class AddHeaderCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("HEADER").SetNone();
+    bool has_side_input = false;
+    bool has_header_stream = false;
+    if (cc->InputSidePackets().HasTag("HEADER")) {
+      cc->InputSidePackets().Tag("HEADER").SetAny();
+      has_side_input = true;
+    }
+    if (cc->Inputs().HasTag("HEADER")) {
+      cc->Inputs().Tag("HEADER").SetNone();
+      has_header_stream = true;
+    }
+    if (has_side_input == has_header_stream) {
+      return mediapipe::InvalidArgumentError(
+          "Header must be provided via exactly one of side input and input "
+          "stream");
+    }
     cc->Inputs().Tag("DATA").SetAny();
     cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    const Packet& header = cc->Inputs().Tag("HEADER").Header();
+    Packet header;
+    if (cc->InputSidePackets().HasTag("HEADER")) {
+      header = cc->InputSidePackets().Tag("HEADER");
+    }
+    if (cc->Inputs().HasTag("HEADER")) {
+      header = cc->Inputs().Tag("HEADER").Header();
+    }
     if (!header.IsEmpty()) {
       cc->Outputs().Index(0).SetHeader(header);
     }

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -14,8 +14,10 @@
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/timestamp.h"
 #include "mediapipe/framework/tool/validate_type.h"
@@ -24,7 +26,7 @@ namespace mediapipe {
 
 class AddHeaderCalculatorTest : public ::testing::Test {};
 
-TEST_F(AddHeaderCalculatorTest, Works) {
+TEST_F(AddHeaderCalculatorTest, HeaderStream) {
   CalculatorGraphConfig::Node node;
   node.set_calculator("AddHeaderCalculator");
   node.add_input_stream("HEADER:header_stream");
@@ -96,4 +98,62 @@ TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
   ASSERT_FALSE(runner.Run().ok());
 }
 
+TEST_F(AddHeaderCalculatorTest, InputSidePacket) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+  node.add_input_side_packet("HEADER:header");
+
+  CalculatorRunner runner(node);
+
+  // Set header and add 5 packets.
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run calculator.
+  MP_ASSERT_OK(runner.Run());
+
+  ASSERT_EQ(1, runner.Outputs().NumEntries());
+
+  // Test output.
+  EXPECT_EQ(std::string("my_header"),
+            runner.Outputs().Index(0).header.Get<std::string>());
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(5, output_packets.size());
+  for (int i = 0; i < 5; ++i) {
+    const int val = output_packets[i].Get<int>();
+    EXPECT_EQ(i, val);
+    EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp());
+  }
+}
+
+TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("HEADER:header_stream");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+  node.add_input_side_packet("HEADER:header");
+
+  CalculatorRunner runner(node);
+
+  // Set both headers and add 5 packets.
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run should fail because header can only be provided one way.
+  EXPECT_EQ(runner.Run().code(), ::mediapipe::InvalidArgumentError("").code());
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -21,16 +21,10 @@
 
 namespace mediapipe {
 
-// A calculator to process std::vector<NormalizedLandmark>.
-typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
-    BeginLoopNormalizedLandmarkCalculator;
-REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);
-
-// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
-typedef BeginLoopCalculator<
-    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
-    BeginLoopNormalizedLandmarksVectorCalculator;
-REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
+// A calculator to process std::vector<NormalizedLandmarkList>.
+typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
+    BeginLoopNormalizedLandmarkListVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator);
 
 // A calculator to process std::vector<NormalizedRect>.
 typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -19,7 +19,7 @@
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
@@ -50,7 +50,7 @@ typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
     ConcatenateLandmarkVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
     ConcatenateGlBufferVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);

mediapipe/calculators/core/end_loop_calculator.cc

@@ -26,14 +26,9 @@ typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
     EndLoopNormalizedRectCalculator;
 REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
 
-typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
-    EndLoopNormalizedLandmarkCalculator;
-REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);
-
-typedef EndLoopCalculator<
-    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
-    EndLoopNormalizedLandmarksVectorCalculator;
-REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
+typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
+    EndLoopNormalizedLandmarkListVectorCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);
 
 typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
 REGISTER_CALCULATOR(EndLoopBooleanCalculator);

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -330,22 +330,27 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
     return ::mediapipe::OkStatus();
   }
 
-  LOG_IF(WARNING, frame_time_usec_ <
-                      (cc->InputTimestamp() - last_packet_.Timestamp()).Value())
-      << "Adding jitter is meaningless when upsampling.";
+  if (frame_time_usec_ <
+      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
 
-  const int64 curr_diff =
-      (next_output_timestamp_ - cc->InputTimestamp()).Value();
-  const int64 last_diff =
-      (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-  if (curr_diff * last_diff > 0) {
-    return ::mediapipe::OkStatus();
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
+    RET_CHECK_GT(last_diff, 0.0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0.0) {
+      break;
+    }
+    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
+                                ? last_packet_
+                                : cc->Inputs().Get(input_data_id_).Value())
+                               .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
   }
-  OutputWithinLimits(cc, (std::abs(curr_diff) > std::abs(last_diff)
-                              ? last_packet_
-                              : cc->Inputs().Get(input_data_id_).Value())
-                             .At(next_output_timestamp_));
-  UpdateNextOutputTimestampWithJitter();
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -74,6 +74,10 @@ class PreviousLoopbackCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) final {
+    Packet& main_packet = cc->Inputs().Get(main_id_).Value();
+    if (!main_packet.IsEmpty()) {
+      main_ts_.push_back(main_packet.Timestamp());
+    }
     Packet& loopback_packet = cc->Inputs().Get(loop_id_).Value();
     if (!loopback_packet.IsEmpty()) {
       loopback_packets_.push_back(loopback_packet);
@@ -83,23 +87,6 @@ class PreviousLoopbackCalculator : public CalculatorBase {
       }
     }
 
-    Packet& main_packet = cc->Inputs().Get(main_id_).Value();
-    if (!main_packet.IsEmpty()) {
-      main_ts_.push_back(main_packet.Timestamp());
-
-      // In case of an empty "LOOP" input, truncate timestamp is set to the
-      // lowest possible timestamp for a successive non-empty "LOOP" input. This
-      // truncates main_ts_ as soon as possible, and produces the highest legal
-      // output timestamp bound.
-      if (loopback_packet.IsEmpty() &&
-          loopback_packet.Timestamp() != Timestamp::Unstarted()) {
-        while (!main_ts_.empty() &&
-               main_ts_.front() <= loopback_packet.Timestamp() + 1) {
-          main_ts_.pop_front();
-        }
-      }
-    }
-
     while (!main_ts_.empty() && !loopback_packets_.empty()) {
       Timestamp main_timestamp = main_ts_.front();
       main_ts_.pop_front();

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -93,14 +93,19 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));
 
+  send_packet("in", 2);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2}));
+  EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(2, 1));
+
   send_packet("in", 5);
   MP_EXPECT_OK(graph_.WaitUntilIdle());
-  EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
-  EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
+  EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5}));
+  EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 2));
 
   send_packet("in", 15);
   MP_EXPECT_OK(graph_.WaitUntilIdle());
-  EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
+  EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 2, 5, 15}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));
 
   MP_EXPECT_OK(graph_.CloseAllInputStreams());
@@ -182,80 +187,25 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));
 
+  send_packet("in", 2);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2}));
+
   send_packet("in", 5);
   MP_EXPECT_OK(graph_.WaitUntilIdle());
-  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5}));
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5}));
 
   send_packet("in", 15);
   MP_EXPECT_OK(graph_.WaitUntilIdle());
-  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5, 15}));
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 2, 5, 15}));
 
   MP_EXPECT_OK(graph_.CloseAllInputStreams());
   MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(outputs),
-            (std::vector<int64>{1, 5, 15, Timestamp::Max().Value()}));
+            (std::vector<int64>{1, 2, 5, 15, Timestamp::Max().Value()}));
 
   MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
-// Demonstrates that downstream calculators won't be blocked by
-// always-empty-LOOP-stream.
-TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
-  std::vector<Packet> outputs;
-  CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
-        input_stream: 'in'
-        node {
-          calculator: 'PreviousLoopbackCalculator'
-          input_stream: 'MAIN:in'
-          input_stream: 'LOOP:previous'
-          input_stream_info: { tag_index: 'LOOP' back_edge: true }
-          output_stream: 'PREV_LOOP:previous'
-        }
-        # This calculator synchronizes its inputs as normal, so it is used
-        # to check that both "in" and "previous" are ready.
-        node {
-          calculator: 'PassThroughCalculator'
-          input_stream: 'in'
-          input_stream: 'previous'
-          output_stream: 'out'
-          output_stream: 'previous2'
-        }
-        node {
-          calculator: 'PacketOnCloseCalculator'
-          input_stream: 'out'
-          output_stream: 'close_out'
-        }
-      )");
-  tool::AddVectorSink("close_out", &graph_config_, &outputs);
-
-  CalculatorGraph graph_;
-  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
-  MP_ASSERT_OK(graph_.StartRun({}));
-
-  auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MP_EXPECT_OK(graph_.AddPacketToInputStream(
-        input_name, MakePacket<int>(n).At(Timestamp(n))));
-  };
-
-  send_packet("in", 0);
-  MP_EXPECT_OK(graph_.WaitUntilIdle());
-  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
-
-  for (int main_ts = 1; main_ts < 50; ++main_ts) {
-    send_packet("in", main_ts);
-    MP_EXPECT_OK(graph_.WaitUntilIdle());
-    std::vector<int64> ts_values = TimestampValues(outputs);
-    EXPECT_EQ(ts_values.size(), main_ts);
-    for (int j = 0; j < main_ts; ++j) {
-      CHECK_EQ(ts_values[j], j);
-    }
-  }
-
-  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MP_EXPECT_OK(graph_.WaitUntilIdle());
-  MP_EXPECT_OK(graph_.WaitUntilDone());
-}
-
 }  // anonymous namespace
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
@@ -41,4 +42,8 @@ REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
 typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
     SplitLandmarkVectorCalculator;
 REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
+
+typedef SplitVectorCalculator<::mediapipe::NormalizedRect>
+    SplitNormalizedRectVectorCalculator;
+REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -356,13 +356,13 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "//mediapipe/gpu:gpu_buffer",
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gl_quad_renderer",
+            "//mediapipe/gpu:gpu_buffer",
             "//mediapipe/gpu:shader_util",
         ],
     }),

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -501,8 +501,11 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
     row_max = std::max(row_max, transformed_points_[i * 2 + 1]);
   }
 
-  *dst_width = std::round((col_max - col_min) * src_width);
-  *dst_height = std::round((row_max - row_min) * src_height);
+  int width = static_cast<int>(std::round((col_max - col_min) * src_width));
+  int height = static_cast<int>(std::round((row_max - row_min) * src_height));
+  // Minimum output dimension 1x1 prevents creation of textures with 0x0.
+  *dst_width = std::max(1, width);
+  *dst_height = std::max(1, height);
 }
 
 }  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -400,7 +400,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   QuadRenderer* renderer = nullptr;
   GlTexture src1;
 
-#if defined(__APPLE__) && !TARGET_OS_OSX
+#if defined(MEDIAPIPE_IOS)
   if (input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange ||
       input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) {
     if (!yuv_renderer_) {

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -34,7 +34,7 @@
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_util.h"
 
-#if !defined(__ANDROID__) && !defined(__APPLE__)
+#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
 #include "tensorflow/core/profiler/lib/traceme.h"
 #endif
 
@@ -441,7 +441,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
     const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
     tf::Status tf_status;
     {
-#if !defined(__ANDROID__) && !defined(__APPLE__)
+#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
       tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
 #endif
       tf_status = session_->Run(input_tensors, output_tensor_names,

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -31,8 +31,7 @@
 #include "mediapipe/framework/tool/status_util.h"
 #include "tensorflow/core/public/session_options.h"
 
-#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
-    defined(__APPLE__) && !TARGET_OS_OSX
+#if defined(MEDIAPIPE_MOBILE)
 #include "mediapipe/util/android/file/base/helpers.h"
 #else
 #include "mediapipe/framework/port/file_helpers.h"

mediapipe/calculators/tflite/BUILD

@@ -13,12 +13,12 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "ssd_anchors_calculator_proto",
     srcs = ["ssd_anchors_calculator.proto"],
@@ -249,6 +249,11 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
         ],
+    }) + select({
+        "//conditions:default": [],
+        "//mediapipe:android": [
+            "@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate",
+        ],
     }),
     alwayslink = 1,
 )

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -25,8 +25,7 @@
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/interpreter.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
@@ -35,7 +34,7 @@
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
 #import <CoreVideo/CoreVideo.h>
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
@@ -46,10 +45,9 @@
 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
 #endif  // iOS
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
 typedef id<MTLBuffer> GpuTensor;
 #endif
 
@@ -69,8 +67,7 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
 
 namespace mediapipe {
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlProgram;
 using ::tflite::gpu::gl::GlShader;
@@ -80,7 +77,7 @@ struct GPUData {
   GlShader shader;
   GlProgram program;
 };
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
 struct GPUData {
   int elements = 1;
   GpuTensor buffer;
@@ -149,11 +146,10 @@ class TfLiteConverterCalculator : public CalculatorBase {
 
   std::unique_ptr<tflite::Interpreter> interpreter_ = nullptr;
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GPUData> gpu_data_out_;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::unique_ptr<GPUData> gpu_data_out_;
 #endif
@@ -202,10 +198,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
   if (use_gpu) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif
   }
@@ -236,10 +231,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
               cc->Outputs().HasTag("TENSORS_GPU"));
     // Cannot use quantization.
     use_quantized_tensors_ = false;
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
     RET_CHECK(gpu_helper_);
 #endif
@@ -270,11 +264,10 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::Close(CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   gpu_helper_.RunInGlContext([this] { gpu_data_out_.reset(); });
 #endif
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
   gpu_data_out_.reset();
 #endif
   return ::mediapipe::OkStatus();
@@ -390,8 +383,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::ProcessGPU(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // GpuBuffer to tflite::gpu::GlBuffer conversion.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
   MP_RETURN_IF_ERROR(
@@ -427,7 +419,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   cc->Outputs()
       .Tag("TENSORS_GPU")
       .Add(output_tensors.release(), cc->InputTimestamp());
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
   {
@@ -493,8 +485,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     RET_CHECK_FAIL() << "Num input channels is less than desired output.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
       [this, &include_alpha, &input, &single_channel]() -> ::mediapipe::Status {
         // Device memory.
@@ -538,7 +529,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
                                                    &gpu_data_out_->program));
         return ::mediapipe::OkStatus();
       }));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+
+#elif defined(MEDIAPIPE_IOS)
+
   RET_CHECK(include_alpha)
       << "iOS GPU inference currently accepts only RGBA input.";
 
@@ -619,7 +612,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   CHECK_GE(max_num_channels_, 1);
   CHECK_LE(max_num_channels_, 4);
   CHECK_NE(max_num_channels_, 2);
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
   if (cc->Inputs().HasTag("IMAGE_GPU"))
     // Currently on iOS, tflite gpu input tensor must be 4 channels,
     // so input image must be 4 channels also (checked in InitGpu).

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -27,7 +27,7 @@
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
@@ -35,9 +35,9 @@
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
-#endif  //  !MEDIAPIPE_DISABLE_GPU
+#endif  //  !MEDIAPIPE_DISABLE_GL_COMPUTE
 
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
 #import <CoreVideo/CoreVideo.h>
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
@@ -51,12 +51,15 @@
 #include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
 #endif  // iOS
 
+#if defined(MEDIAPIPE_ANDROID)
+#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
+#endif  // ANDROID
+
 namespace {
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
 typedef id<MTLBuffer> GpuTensor;
 #endif
 
@@ -92,8 +95,7 @@ std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
 //  * Aux
 namespace mediapipe {
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 using ::tflite::gpu::gl::CopyBuffer;
 using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlBuffer;
@@ -172,12 +174,11 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   std::unique_ptr<tflite::FlatBufferModel> model_;
   TfLiteDelegate* delegate_ = nullptr;
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GPUData> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::unique_ptr<GPUData> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
@@ -237,10 +238,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   use_gpu |= options.use_gpu();
 
   if (use_gpu) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif
   }
@@ -280,26 +280,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   MP_RETURN_IF_ERROR(LoadModel(cc));
 
   if (gpu_inference_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
     RET_CHECK(gpu_helper_);
 #endif
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
 #else
     MP_RETURN_IF_ERROR(LoadDelegate(cc));
 #endif
+  } else {
+#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_ANDROID)
+    MP_RETURN_IF_ERROR(LoadDelegate(cc));
+#endif  // __EMSCRIPTEN__ || ANDROID
   }
-
-#if defined(__EMSCRIPTEN__)
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
-#endif  // __EMSCRIPTEN__
-
   return ::mediapipe::OkStatus();
 }
 
@@ -307,8 +305,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   // 1. Receive pre-processed tensor inputs.
   if (gpu_input_) {
     // Read GPU input into SSBO.
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
     RET_CHECK_EQ(input_tensors.size(), 1);
@@ -318,7 +315,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           RET_CHECK_CALL(CopyBuffer(input_tensors[0], gpu_data_in_->buffer));
           return ::mediapipe::OkStatus();
         }));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
     RET_CHECK_EQ(input_tensors.size(), 1);
@@ -354,14 +351,13 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // 2. Run inference.
   if (gpu_inference_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
           RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
           return ::mediapipe::OkStatus();
         }));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
 #endif
   } else {
@@ -370,8 +366,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // 3. Output processed tensors.
   if (gpu_output_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
@@ -388,7 +383,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     cc->Outputs()
         .Tag("TENSORS_GPU")
         .Add(output_tensors.release(), cc->InputTimestamp());
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
     output_tensors->resize(gpu_data_out_.size());
@@ -433,23 +428,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
   if (delegate_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
-      TfLiteGpuDelegateDelete(delegate_);
+    if (gpu_inference_) {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+      MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
+        TfLiteGpuDelegateDelete(delegate_);
+        gpu_data_in_.reset();
+        for (int i = 0; i < gpu_data_out_.size(); ++i) {
+          gpu_data_out_[i].reset();
+        }
+        return ::mediapipe::OkStatus();
+      }));
+#elif defined(MEDIAPIPE_IOS)
+      TFLGpuDelegateDelete(delegate_);
       gpu_data_in_.reset();
       for (int i = 0; i < gpu_data_out_.size(); ++i) {
         gpu_data_out_[i].reset();
       }
-      return ::mediapipe::OkStatus();
-    }));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
-    TFLGpuDelegateDelete(delegate_);
-    gpu_data_in_.reset();
-    for (int i = 0; i < gpu_data_out_.size(); ++i) {
-      gpu_data_out_[i].reset();
-    }
 #endif
+    }
     delegate_ = nullptr;
   }
 #if defined(MEDIAPIPE_EDGE_TPU)
@@ -468,7 +464,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // Get model name.
   if (!options.model_path().empty()) {
-    auto model_path = options.model_path();
+    std::string model_path = options.model_path();
 
     ASSIGN_OR_RETURN(model_path_, mediapipe::PathToResourceAsFile(model_path));
   } else {
@@ -522,8 +518,22 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if defined(MEDIAPIPE_ANDROID)
+  if (!gpu_inference_) {
+    if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
+            .use_nnapi()) {
+      // Attempt to use NNAPI.
+      // If not supported, the default CPU delegate will be created and used.
+      interpreter_->SetAllowFp16PrecisionForFp32(1);
+      delegate_ = tflite::NnApiDelegate();
+      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+    }
+    // Return, no need for GPU delegate below.
+    return ::mediapipe::OkStatus();
+  }
+#endif  // ANDROID
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Configure and create the delegate.
   TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
   options.compile_options.precision_loss_allowed = 1;
@@ -583,7 +593,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
 #endif  // OpenGL
 
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
   // Configure and create the delegate.
   TFLGpuDelegateOptions options;
   options.allow_precision_loss = false;  // Must match converter, F=float/T=half

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -45,4 +45,9 @@ message TfLiteInferenceCalculatorOptions {
   // input tensors are on CPU. For input tensors on GPU, GPU backend is always
   // used.
   optional bool use_gpu = 2 [default = false];
+
+  // Android only. When true, an NNAPI delegate will be used for inference.
+  // If NNAPI is not available, then the default CPU delegate will be used
+  // automatically.
+  optional bool use_nnapi = 3 [default = false];
 }

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc

@@ -24,8 +24,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/interpreter.h"
-#if defined(__EMSCRIPTEN__) || defined(__ANDROID__) || \
-    (defined(__APPLE__) && !TARGET_OS_OSX)
+#if defined(MEDIAPIPE_MOBILE)
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/helpers.h"
 #else

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -27,8 +27,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "tensorflow/lite/interpreter.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
@@ -36,7 +35,7 @@
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-#if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#if defined(MEDIAPIPE_IOS)
 #import <CoreVideo/CoreVideo.h>
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
@@ -56,17 +55,15 @@ constexpr int kNumCoordsPerBox = 4;
 
 namespace mediapipe {
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlShader;
 #endif
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
 typedef ::tflite::gpu::gl::GlProgram GpuProgram;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
 typedef id<MTLBuffer> GpuTensor;
 typedef id<MTLComputePipelineState> GpuProgram;
 #endif
@@ -183,11 +180,10 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
   std::vector<Anchor> anchors_;
   bool side_packet_anchors_{};
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GPUData> gpu_data_;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::unique_ptr<GPUData> gpu_data_;
 #endif
@@ -226,10 +222,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
   }
 
   if (use_gpu) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif
   }
@@ -243,10 +238,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
   if (cc->Inputs().HasTag("TENSORS_GPU")) {
     gpu_input_ = true;
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
     gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
     RET_CHECK(gpu_helper_);
 #endif
@@ -406,8 +400,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 }
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   const auto& input_tensors =
       cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
@@ -470,7 +463,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
     return ::mediapipe::OkStatus();
   }));
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
 
   const auto& input_tensors =
       cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
@@ -569,12 +562,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::Close(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   gpu_helper_.RunInGlContext([this] { gpu_data_.reset(); });
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
+#elif defined(MEDIAPIPE_IOS)
   gpu_data_.reset();
-#endif                                      //  !MEDIAPIPE_DISABLE_GPU
+#endif
 
   return ::mediapipe::OkStatus();
 }
@@ -723,8 +715,7 @@ Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection(
 
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::GpuInit(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]()
                                                     -> ::mediapipe::Status {
     gpu_data_ = absl::make_unique<GPUData>();
@@ -937,8 +928,7 @@ void main() {
     return ::mediapipe::OkStatus();
   }));
 
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
-  // TODO consolidate Metal and OpenGL shaders via vulkan.
+#elif defined(MEDIAPIPE_IOS)
 
   gpu_data_ = absl::make_unique<GPUData>();
   id<MTLDevice> device = gpu_helper_.mtlDevice;
@@ -1168,7 +1158,7 @@ kernel void scoreKernel(
     CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
   }
 
-#endif  // __ANDROID__ or iOS
+#endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -76,11 +76,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
   }
 
   if (cc->Outputs().HasTag("LANDMARKS")) {
-    cc->Outputs().Tag("LANDMARKS").Set<std::vector<Landmark>>();
+    cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
   }
 
   if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
-    cc->Outputs().Tag("NORM_LANDMARKS").Set<std::vector<NormalizedLandmark>>();
+    cc->Outputs().Tag("NORM_LANDMARKS").Set<NormalizedLandmarkList>();
   }
 
   return ::mediapipe::OkStatus();
@@ -127,54 +127,55 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
 
   const float* raw_landmarks = raw_tensor->data.f;
 
-  auto output_landmarks = absl::make_unique<std::vector<Landmark>>();
+  LandmarkList output_landmarks;
 
   for (int ld = 0; ld < num_landmarks_; ++ld) {
     const int offset = ld * num_dimensions;
-    Landmark landmark;
+    Landmark* landmark = output_landmarks.add_landmark();
 
     if (options_.flip_horizontally()) {
-      landmark.set_x(options_.input_image_width() - raw_landmarks[offset]);
+      landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
     } else {
-      landmark.set_x(raw_landmarks[offset]);
+      landmark->set_x(raw_landmarks[offset]);
     }
     if (num_dimensions > 1) {
       if (options_.flip_vertically()) {
-        landmark.set_y(options_.input_image_height() -
-                       raw_landmarks[offset + 1]);
+        landmark->set_y(options_.input_image_height() -
+                        raw_landmarks[offset + 1]);
       } else {
-        landmark.set_y(raw_landmarks[offset + 1]);
+        landmark->set_y(raw_landmarks[offset + 1]);
       }
     }
     if (num_dimensions > 2) {
-      landmark.set_z(raw_landmarks[offset + 2]);
+      landmark->set_z(raw_landmarks[offset + 2]);
     }
-    output_landmarks->push_back(landmark);
   }
 
   // Output normalized landmarks if required.
   if (cc->Outputs().HasTag("NORM_LANDMARKS")) {
-    auto output_norm_landmarks =
-        absl::make_unique<std::vector<NormalizedLandmark>>();
-    for (const auto& landmark : *output_landmarks) {
-      NormalizedLandmark norm_landmark;
-      norm_landmark.set_x(static_cast<float>(landmark.x()) /
-                          options_.input_image_width());
-      norm_landmark.set_y(static_cast<float>(landmark.y()) /
-                          options_.input_image_height());
-      norm_landmark.set_z(landmark.z() / options_.normalize_z());
-
-      output_norm_landmarks->push_back(norm_landmark);
+    NormalizedLandmarkList output_norm_landmarks;
+    // for (const auto& landmark : output_landmarks) {
+    for (int i = 0; i < output_landmarks.landmark_size(); ++i) {
+      const Landmark& landmark = output_landmarks.landmark(i);
+      NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark();
+      norm_landmark->set_x(static_cast<float>(landmark.x()) /
+                           options_.input_image_width());
+      norm_landmark->set_y(static_cast<float>(landmark.y()) /
+                           options_.input_image_height());
+      norm_landmark->set_z(landmark.z() / options_.normalize_z());
     }
     cc->Outputs()
         .Tag("NORM_LANDMARKS")
-        .Add(output_norm_landmarks.release(), cc->InputTimestamp());
+        .AddPacket(MakePacket<NormalizedLandmarkList>(output_norm_landmarks)
+                       .At(cc->InputTimestamp()));
   }
+
   // Output absolute landmarks.
   if (cc->Outputs().HasTag("LANDMARKS")) {
     cc->Outputs()
         .Tag("LANDMARKS")
-        .Add(output_landmarks.release(), cc->InputTimestamp());
+        .AddPacket(MakePacket<LandmarkList>(output_landmarks)
+                       .At(cc->InputTimestamp()));
   }
 
   return ::mediapipe::OkStatus();

mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc

@@ -28,8 +28,7 @@
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/interpreter.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/shader_util.h"
@@ -54,8 +53,7 @@ float Clamp(float val, float min, float max) {
 
 namespace mediapipe {
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 using ::tflite::gpu::gl::CopyBuffer;
 using ::tflite::gpu::gl::CreateReadWriteRgbaImageTexture;
 using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
@@ -131,8 +129,7 @@ class TfLiteTensorsToSegmentationCalculator : public CalculatorBase {
   int tensor_channels_ = 0;
 
   bool use_gpu_ = false;
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GlProgram> mask_program_with_prev_;
   std::unique_ptr<GlProgram> mask_program_no_prev_;
@@ -162,8 +159,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   }
 
   // Inputs GPU.
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   if (cc->Inputs().HasTag("TENSORS_GPU")) {
     cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GlBuffer>>();
     use_gpu |= true;
@@ -182,8 +178,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   if (cc->Outputs().HasTag("MASK")) {
     cc->Outputs().Tag("MASK").Set<ImageFrame>();
   }
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   if (cc->Outputs().HasTag("MASK_GPU")) {
     cc->Outputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
@@ -191,8 +186,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
   if (use_gpu) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
@@ -205,8 +199,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 
   if (cc->Inputs().HasTag("TENSORS_GPU")) {
     use_gpu_ = true;
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
@@ -214,8 +207,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   MP_RETURN_IF_ERROR(LoadOptions(cc));
 
   if (use_gpu_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
           MP_RETURN_IF_ERROR(InitGpu(cc));
@@ -232,8 +224,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::Process(
     CalculatorContext* cc) {
   if (use_gpu_) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
           MP_RETURN_IF_ERROR(ProcessGpu(cc));
@@ -249,8 +240,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::Close(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   gpu_helper_.RunInGlContext([this] {
     if (upsample_program_) glDeleteProgram(upsample_program_);
     upsample_program_ = 0;
@@ -377,8 +367,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   if (cc->Inputs().Tag("TENSORS_GPU").IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Get input streams.
   const auto& input_tensors =
       cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GlBuffer>>();
@@ -464,8 +453,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 }
 
 void TfLiteTensorsToSegmentationCalculator::GlRender() {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -537,8 +525,7 @@ void TfLiteTensorsToSegmentationCalculator::GlRender() {
 
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::InitGpu(
     CalculatorContext* cc) {
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    !defined(__APPLE__)
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]()
                                                     -> ::mediapipe::Status {
     // A shader to process a segmentation tensor into an output mask,

mediapipe/calculators/util/BUILD

@@ -983,3 +983,31 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
     ],
 )
+
+cc_library(
+    name = "detections_to_timed_box_list_calculator",
+    srcs = ["detections_to_timed_box_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:box_tracker",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "detection_unique_id_calculator",
+    srcs = ["detection_unique_id_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/util/detection_label_id_to_text_calculator.cc

@@ -19,8 +19,7 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/resource_util.h"
 
-#if defined(MEDIAPIPE_LITE) || defined(__EMSCRIPTEN__) || \
-    defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if defined(MEDIAPIPE_MOBILE)
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/helpers.h"
 #else

mediapipe/calculators/util/detection_unique_id_calculator.cc

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+
+// Each detection processed by DetectionUniqueIDCalculator will be assigned an
+// unique id that starts from 1. If a detection already has an ID other than 0,
+// the ID will be overwritten.
+static int64 detection_id = 0;
+
+inline int GetNextDetectionId() { return ++detection_id; }
+
+}  // namespace
+
+// Assign a unique id to detections.
+// Note that the calculator will consume the input vector of Detection or
+// DetectionList. So the input stream can not be connected to other calculators.
+//
+// Example config:
+// node {
+//   calculator: "DetectionUniqueIdCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "DETECTIONS:output_detections"
+// }
+class DetectionUniqueIdCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
+              cc->Inputs().HasTag(kDetectionsTag))
+        << "None of the input streams are provided.";
+
+    if (cc->Inputs().HasTag(kDetectionListTag)) {
+      RET_CHECK(cc->Outputs().HasTag(kDetectionListTag));
+      cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+      cc->Outputs().Tag(kDetectionListTag).Set<DetectionList>();
+    }
+    if (cc->Inputs().HasTag(kDetectionsTag)) {
+      RET_CHECK(cc->Outputs().HasTag(kDetectionsTag));
+      cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+      cc->Outputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(::mediapipe::TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(DetectionUniqueIdCalculator);
+
+::mediapipe::Status DetectionUniqueIdCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kDetectionListTag) &&
+      !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) {
+    auto result =
+        cc->Inputs().Tag(kDetectionListTag).Value().Consume<DetectionList>();
+    if (result.ok()) {
+      auto detection_list = std::move(result).ValueOrDie();
+      for (Detection& detection : *detection_list->mutable_detection()) {
+        detection.set_detection_id(GetNextDetectionId());
+      }
+      cc->Outputs()
+          .Tag(kDetectionListTag)
+          .Add(detection_list.release(), cc->InputTimestamp());
+    }
+  }
+
+  if (cc->Inputs().HasTag(kDetectionsTag) &&
+      !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) {
+    auto result = cc->Inputs()
+                      .Tag(kDetectionsTag)
+                      .Value()
+                      .Consume<std::vector<Detection>>();
+    if (result.ok()) {
+      auto detections = std::move(result).ValueOrDie();
+      for (Detection& detection : *detections) {
+        detection.set_detection_id(GetNextDetectionId());
+      }
+      cc->Outputs()
+          .Tag(kDetectionsTag)
+          .Add(detections.release(), cc->InputTimestamp());
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/detections_to_render_data_calculator.cc

@@ -39,6 +39,8 @@ constexpr char kKeypointLabel[] = "KEYPOINT";
 // The ratio of detection label font height to the height of detection bounding
 // box.
 constexpr double kLabelToBoundingBoxRatio = 0.1;
+// Perserve 2 decimal digits.
+constexpr float kNumScoreDecimalDigitsMultipler = 100;
 
 }  // namespace
 
@@ -235,18 +237,26 @@ void DetectionsToRenderDataCalculator::AddLabels(
     std::string label_str = detection.label().empty()
                                 ? absl::StrCat(detection.label_id(i))
                                 : detection.label(i);
+    const float rounded_score =
+        std::round(detection.score(i) * kNumScoreDecimalDigitsMultipler) /
+        kNumScoreDecimalDigitsMultipler;
     std::string label_and_score =
-        absl::StrCat(label_str, options.text_delimiter(), detection.score(i),
+        absl::StrCat(label_str, options.text_delimiter(), rounded_score,
                      options.text_delimiter());
     label_and_scores.push_back(label_and_score);
   }
   std::vector<std::string> labels;
+  if (options.render_detection_id()) {
+    const std::string detection_id_str =
+        absl::StrCat("Id: ", detection.detection_id());
+    labels.push_back(detection_id_str);
+  }
   if (options.one_label_per_line()) {
-    labels.swap(label_and_scores);
+    labels.insert(labels.end(), label_and_scores.begin(),
+                  label_and_scores.end());
   } else {
     labels.push_back(absl::StrJoin(label_and_scores, ""));
   }
-
   // Add the render annotations for "label(_id),score".
   for (int i = 0; i < labels.size(); ++i) {
     auto label = labels.at(i);

mediapipe/calculators/util/detections_to_render_data_calculator.proto

@@ -53,4 +53,7 @@ message DetectionsToRenderDataCalculatorOptions {
   // instances of this calculator are present in the graph, this value
   // should be unique among them.
   optional string scene_class = 7 [default = "DETECTION"];
+
+  // If true, renders the detection id in the first line before the labels.
+  optional bool render_detection_id = 8 [default = false];
 }

mediapipe/calculators/util/detections_to_timed_box_list_calculator.cc

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+constexpr char kBoxesTag[] = "BOXES";
+
+}  // namespace
+
+// A calculator that converts Detection proto to TimedBoxList proto for
+// tracking.
+//
+// Please note that only Location Data formats of RELATIVE_BOUNDING_BOX are
+// supported.
+//
+// Example config:
+// node {
+//   calculator: "DetectionsToTimedBoxListCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "BOXES:boxes"
+// }
+class DetectionsToTimedBoxListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
+              cc->Inputs().HasTag(kDetectionsTag))
+        << "None of the input streams are provided.";
+    if (cc->Inputs().HasTag(kDetectionListTag)) {
+      cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+    }
+    if (cc->Inputs().HasTag(kDetectionsTag)) {
+      cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+    }
+    cc->Outputs().Tag(kBoxesTag).Set<TimedBoxProtoList>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void ConvertDetectionToTimedBox(const Detection& detection,
+                                  TimedBoxProto* box, CalculatorContext* cc);
+};
+REGISTER_CALCULATOR(DetectionsToTimedBoxListCalculator);
+
+::mediapipe::Status DetectionsToTimedBoxListCalculator::Process(
+    CalculatorContext* cc) {
+  auto output_timed_box_list = absl::make_unique<TimedBoxProtoList>();
+
+  if (cc->Inputs().HasTag(kDetectionListTag)) {
+    const auto& detection_list =
+        cc->Inputs().Tag(kDetectionListTag).Get<DetectionList>();
+    for (const auto& detection : detection_list.detection()) {
+      TimedBoxProto* box = output_timed_box_list->add_box();
+      ConvertDetectionToTimedBox(detection, box, cc);
+    }
+  }
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    const auto& detections =
+        cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>();
+    for (const auto& detection : detections) {
+      TimedBoxProto* box = output_timed_box_list->add_box();
+      ConvertDetectionToTimedBox(detection, box, cc);
+    }
+  }
+
+  cc->Outputs().Tag(kBoxesTag).Add(output_timed_box_list.release(),
+                                   cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+void DetectionsToTimedBoxListCalculator::ConvertDetectionToTimedBox(
+    const Detection& detection, TimedBoxProto* box, CalculatorContext* cc) {
+  const auto& relative_bounding_box =
+      detection.location_data().relative_bounding_box();
+  box->set_left(relative_bounding_box.xmin());
+  box->set_right(relative_bounding_box.xmin() + relative_bounding_box.width());
+  box->set_top(relative_bounding_box.ymin());
+  box->set_bottom(relative_bounding_box.ymin() +
+                  relative_bounding_box.height());
+  box->set_id(detection.detection_id());
+  box->set_time_msec(cc->InputTimestamp().Microseconds() / 1000);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/filter_collection_calculator.cc

@@ -27,8 +27,8 @@ typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
 REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
 
 typedef FilterCollectionCalculator<
-    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
-    FilterLandmarksCollectionCalculator;
-REGISTER_CALCULATOR(FilterLandmarksCollectionCalculator);
+    std::vector<::mediapipe::NormalizedLandmarkList>>
+    FilterLandmarkListCollectionCalculator;
+REGISTER_CALCULATOR(FilterLandmarkListCollectionCalculator);
 
 }  // namespace mediapipe

mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc

@@ -49,7 +49,7 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
 // corresponding input image before letterboxing.
 //
 // Input:
-//   LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks on an
+//   LANDMARKS: A NormalizedLandmarkList representing landmarks on an
 //   letterboxed image.
 //
 //   LETTERBOX_PADDING: An std::array<float, 4> representing the letterbox
@@ -57,7 +57,7 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
 //   image, normalized to [0.f, 1.f] by the letterboxed image dimensions.
 //
 // Output:
-//   LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks with
+//   LANDMARKS: An NormalizedLandmarkList proto representing landmarks with
 //   their locations adjusted to the letterbox-removed (non-padded) image.
 //
 // Usage example:
@@ -74,10 +74,10 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
               cc->Inputs().HasTag(kLetterboxPaddingTag))
         << "Missing one or more input streams.";
 
-    cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
     cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
 
-    cc->Outputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
 
     return ::mediapipe::OkStatus();
   }
@@ -94,8 +94,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
       return ::mediapipe::OkStatus();
     }
 
-    const auto& input_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
+    const NormalizedLandmarkList& input_landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
     const auto& letterbox_padding =
         cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
 
@@ -104,24 +104,23 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
     const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
     const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
 
-    auto output_landmarks =
-        absl::make_unique<std::vector<NormalizedLandmark>>();
-    for (const auto& landmark : input_landmarks) {
-      NormalizedLandmark new_landmark;
+    NormalizedLandmarkList output_landmarks;
+    for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
+      const NormalizedLandmark& landmark = input_landmarks.landmark(i);
+      NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
       const float new_x = (landmark.x() - left) / (1.0f - left_and_right);
       const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom);
 
-      new_landmark.set_x(new_x);
-      new_landmark.set_y(new_y);
+      new_landmark->set_x(new_x);
+      new_landmark->set_y(new_y);
       // Keep z-coord as is.
-      new_landmark.set_z(landmark.z());
-
-      output_landmarks->emplace_back(new_landmark);
+      new_landmark->set_z(landmark.z());
     }
 
     cc->Outputs()
         .Tag(kLandmarksTag)
-        .Add(output_landmarks.release(), cc->InputTimestamp());
+        .AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
+                       .At(cc->InputTimestamp()));
     return ::mediapipe::OkStatus();
   }
 };

mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc

@@ -43,10 +43,10 @@ CalculatorGraphConfig::Node GetDefaultNode() {
 TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {
   CalculatorRunner runner(GetDefaultNode());
 
-  auto landmarks = absl::make_unique<std::vector<NormalizedLandmark>>();
-  landmarks->push_back(CreateLandmark(0.5f, 0.5f));
-  landmarks->push_back(CreateLandmark(0.2f, 0.2f));
-  landmarks->push_back(CreateLandmark(0.7f, 0.7f));
+  auto landmarks = absl::make_unique<NormalizedLandmarkList>();
+  *landmarks->add_landmark() = CreateLandmark(0.5f, 0.5f);
+  *landmarks->add_landmark() = CreateLandmark(0.2f, 0.2f);
+  *landmarks->add_landmark() = CreateLandmark(0.7f, 0.7f);
   runner.MutableInputs()
       ->Tag("LANDMARKS")
       .packets.push_back(
@@ -61,26 +61,28 @@ TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {
   MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& output = runner.Outputs().Tag("LANDMARKS").packets;
   ASSERT_EQ(1, output.size());
-  const auto& output_landmarks =
-      output[0].Get<std::vector<NormalizedLandmark>>();
+  const auto& output_landmarks = output[0].Get<NormalizedLandmarkList>();
 
-  EXPECT_EQ(output_landmarks.size(), 3);
+  EXPECT_EQ(output_landmarks.landmark_size(), 3);
 
-  EXPECT_THAT(output_landmarks[0].x(), testing::FloatNear(0.6f, 1e-5));
-  EXPECT_THAT(output_landmarks[0].y(), testing::FloatNear(0.5f, 1e-5));
-  EXPECT_THAT(output_landmarks[1].x(), testing::FloatNear(0.0f, 1e-5));
-  EXPECT_THAT(output_landmarks[1].y(), testing::FloatNear(0.2f, 1e-5));
-  EXPECT_THAT(output_landmarks[2].x(), testing::FloatNear(1.0f, 1e-5));
-  EXPECT_THAT(output_landmarks[2].y(), testing::FloatNear(0.7f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.6f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.5f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.0f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.2f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(1.0f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(0.7f, 1e-5));
 }
 
 TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingTopBottom) {
   CalculatorRunner runner(GetDefaultNode());
 
-  auto landmarks = absl::make_unique<std::vector<NormalizedLandmark>>();
-  landmarks->push_back(CreateLandmark(0.5f, 0.5f));
-  landmarks->push_back(CreateLandmark(0.2f, 0.2f));
-  landmarks->push_back(CreateLandmark(0.7f, 0.7f));
+  auto landmarks = absl::make_unique<NormalizedLandmarkList>();
+  NormalizedLandmark* landmark = landmarks->add_landmark();
+  *landmark = CreateLandmark(0.5f, 0.5f);
+  landmark = landmarks->add_landmark();
+  *landmark = CreateLandmark(0.2f, 0.2f);
+  landmark = landmarks->add_landmark();
+  *landmark = CreateLandmark(0.7f, 0.7f);
   runner.MutableInputs()
       ->Tag("LANDMARKS")
       .packets.push_back(
@@ -95,17 +97,16 @@ TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingTopBottom) {
   MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& output = runner.Outputs().Tag("LANDMARKS").packets;
   ASSERT_EQ(1, output.size());
-  const auto& output_landmarks =
-      output[0].Get<std::vector<NormalizedLandmark>>();
+  const auto& output_landmarks = output[0].Get<NormalizedLandmarkList>();
 
-  EXPECT_EQ(output_landmarks.size(), 3);
+  EXPECT_EQ(output_landmarks.landmark_size(), 3);
 
-  EXPECT_THAT(output_landmarks[0].x(), testing::FloatNear(0.5f, 1e-5));
-  EXPECT_THAT(output_landmarks[0].y(), testing::FloatNear(0.6f, 1e-5));
-  EXPECT_THAT(output_landmarks[1].x(), testing::FloatNear(0.2f, 1e-5));
-  EXPECT_THAT(output_landmarks[1].y(), testing::FloatNear(0.0f, 1e-5));
-  EXPECT_THAT(output_landmarks[2].x(), testing::FloatNear(0.7f, 1e-5));
-  EXPECT_THAT(output_landmarks[2].y(), testing::FloatNear(1.0f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.5f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.6f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.2f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.0f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(0.7f, 1e-5));
+  EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(1.0f, 1e-5));
 }
 
 }  // namespace mediapipe

mediapipe/calculators/util/landmark_projection_calculator.cc

@@ -47,13 +47,13 @@ constexpr char kRectTag[] = "NORM_RECT";
 // Projects normalized landmarks in a rectangle to its original coordinates. The
 // rectangle must also be in normalized coordinates.
 // Input:
-//   NORM_LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks
+//   NORM_LANDMARKS: A NormalizedLandmarkList representing landmarks
 //                   in a normalized rectangle.
 //   NORM_RECT: An NormalizedRect representing a normalized rectangle in image
 //              coordinates.
 //
 // Output:
-//   NORM_LANDMARKS: An std::vector<NormalizedLandmark> representing landmarks
+//   NORM_LANDMARKS: A NormalizedLandmarkList representing landmarks
 //                   with their locations adjusted to the image.
 //
 // Usage example:
@@ -70,10 +70,10 @@ class LandmarkProjectionCalculator : public CalculatorBase {
               cc->Inputs().HasTag(kRectTag))
         << "Missing one or more input streams.";
 
-    cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
     cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
 
-    cc->Outputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
 
     return ::mediapipe::OkStatus();
   }
@@ -92,14 +92,14 @@ class LandmarkProjectionCalculator : public CalculatorBase {
       return ::mediapipe::OkStatus();
     }
 
-    const auto& input_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
+    const NormalizedLandmarkList& input_landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
     const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
 
-    auto output_landmarks =
-        absl::make_unique<std::vector<NormalizedLandmark>>();
-    for (const auto& landmark : input_landmarks) {
-      NormalizedLandmark new_landmark;
+    NormalizedLandmarkList output_landmarks;
+    for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
+      const NormalizedLandmark& landmark = input_landmarks.landmark(i);
+      NormalizedLandmark* new_landmark = output_landmarks.add_landmark();
 
       const float x = landmark.x() - 0.5f;
       const float y = landmark.y() - 0.5f;
@@ -110,17 +110,16 @@ class LandmarkProjectionCalculator : public CalculatorBase {
       new_x = new_x * input_rect.width() + input_rect.x_center();
       new_y = new_y * input_rect.height() + input_rect.y_center();
 
-      new_landmark.set_x(new_x);
-      new_landmark.set_y(new_y);
+      new_landmark->set_x(new_x);
+      new_landmark->set_y(new_y);
       // Keep z-coord as is.
-      new_landmark.set_z(landmark.z());
-
-      output_landmarks->emplace_back(new_landmark);
+      new_landmark->set_z(landmark.z());
     }
 
     cc->Outputs()
         .Tag(kLandmarksTag)
-        .Add(output_landmarks.release(), cc->InputTimestamp());
+        .AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
+                       .At(cc->InputTimestamp()));
     return ::mediapipe::OkStatus();
   }
 };

mediapipe/calculators/util/landmarks_to_detection_calculator.cc

@@ -28,8 +28,7 @@ namespace {
 constexpr char kDetectionTag[] = "DETECTION";
 constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
 
-Detection ConvertLandmarksToDetection(
-    const std::vector<NormalizedLandmark>& landmarks) {
+Detection ConvertLandmarksToDetection(const NormalizedLandmarkList& landmarks) {
   Detection detection;
   LocationData* location_data = detection.mutable_location_data();
 
@@ -37,7 +36,8 @@ Detection ConvertLandmarksToDetection(
   float x_max = std::numeric_limits<float>::min();
   float y_min = std::numeric_limits<float>::max();
   float y_max = std::numeric_limits<float>::min();
-  for (const auto& landmark : landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const NormalizedLandmark& landmark = landmarks.landmark(i);
     x_min = std::min(x_min, landmark.x());
     x_max = std::max(x_max, landmark.x());
     y_min = std::min(y_min, landmark.y());
@@ -67,7 +67,7 @@ Detection ConvertLandmarksToDetection(
 // to specify a subset of landmarks for creating the detection.
 //
 // Input:
-//  NOMR_LANDMARKS: A vector of NormalizedLandmark.
+//  NOMR_LANDMARKS: A NormalizedLandmarkList proto.
 //
 // Output:
 //   DETECTION: A Detection proto.
@@ -95,9 +95,7 @@ REGISTER_CALCULATOR(LandmarksToDetectionCalculator);
   RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag));
   RET_CHECK(cc->Outputs().HasTag(kDetectionTag));
   // TODO: Also support converting Landmark to Detection.
-  cc->Inputs()
-      .Tag(kNormalizedLandmarksTag)
-      .Set<std::vector<NormalizedLandmark>>();
+  cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
   cc->Outputs().Tag(kDetectionTag).Set<Detection>();
 
   return ::mediapipe::OkStatus();
@@ -113,19 +111,20 @@ REGISTER_CALCULATOR(LandmarksToDetectionCalculator);
 
 ::mediapipe::Status LandmarksToDetectionCalculator::Process(
     CalculatorContext* cc) {
-  const auto& landmarks = cc->Inputs()
-                              .Tag(kNormalizedLandmarksTag)
-                              .Get<std::vector<NormalizedLandmark>>();
-  RET_CHECK_GT(landmarks.size(), 0) << "Input landmark vector is empty.";
+  const auto& landmarks =
+      cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
+  RET_CHECK_GT(landmarks.landmark_size(), 0)
+      << "Input landmark vector is empty.";
 
   auto detection = absl::make_unique<Detection>();
   if (options_.selected_landmark_indices_size()) {
-    std::vector<NormalizedLandmark> subset_landmarks(
-        options_.selected_landmark_indices_size());
-    for (int i = 0; i < subset_landmarks.size(); ++i) {
-      RET_CHECK_LT(options_.selected_landmark_indices(i), landmarks.size())
+    NormalizedLandmarkList subset_landmarks;
+    for (int i = 0; i < options_.selected_landmark_indices_size(); ++i) {
+      RET_CHECK_LT(options_.selected_landmark_indices(i),
+                   landmarks.landmark_size())
           << "Index of landmark subset is out of range.";
-      subset_landmarks[i] = landmarks[options_.selected_landmark_indices(i)];
+      *subset_landmarks.add_landmark() =
+          landmarks.landmark(options_.selected_landmark_indices(i));
     }
     *detection = ConvertLandmarksToDetection(subset_landmarks);
   } else {

mediapipe/calculators/util/landmarks_to_floats_calculator.cc

@@ -48,7 +48,7 @@ constexpr char kMatrixTag[] = "MATRIX";
 
 // Converts a vector of landmarks to a vector of floats or a matrix.
 // Input:
-//   NORM_LANDMARKS: An std::vector<NormalizedLandmark>.
+//   NORM_LANDMARKS: A NormalizedLandmarkList proto.
 //
 // Output:
 //   FLOATS(optional): A vector of floats from flattened landmarks.
@@ -63,7 +63,7 @@ constexpr char kMatrixTag[] = "MATRIX";
 class LandmarksToFloatsCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag(kLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
     RET_CHECK(cc->Outputs().HasTag(kFloatsTag) ||
               cc->Outputs().HasTag(kMatrixTag));
     if (cc->Outputs().HasTag(kFloatsTag)) {
@@ -94,11 +94,12 @@ class LandmarksToFloatsCalculator : public CalculatorBase {
     }
 
     const auto& input_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<std::vector<NormalizedLandmark>>();
+        cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
 
     if (cc->Outputs().HasTag(kFloatsTag)) {
       auto output_floats = absl::make_unique<std::vector<float>>();
-      for (const auto& landmark : input_landmarks) {
+      for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
+        const NormalizedLandmark& landmark = input_landmarks.landmark(i);
         output_floats->emplace_back(landmark.x());
         if (num_dimensions_ > 1) {
           output_floats->emplace_back(landmark.y());
@@ -113,14 +114,14 @@ class LandmarksToFloatsCalculator : public CalculatorBase {
           .Add(output_floats.release(), cc->InputTimestamp());
     } else {
       auto output_matrix = absl::make_unique<Matrix>();
-      output_matrix->setZero(num_dimensions_, input_landmarks.size());
-      for (int i = 0; i < input_landmarks.size(); ++i) {
-        (*output_matrix)(0, i) = input_landmarks[i].x();
+      output_matrix->setZero(num_dimensions_, input_landmarks.landmark_size());
+      for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
+        (*output_matrix)(0, i) = input_landmarks.landmark(i).x();
         if (num_dimensions_ > 1) {
-          (*output_matrix)(1, i) = input_landmarks[i].y();
+          (*output_matrix)(1, i) = input_landmarks.landmark(i).y();
         }
         if (num_dimensions_ > 2) {
-          (*output_matrix)(2, i) = input_landmarks[i].z();
+          (*output_matrix)(2, i) = input_landmarks.landmark(i).z();
         }
       }
       cc->Outputs()

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -46,12 +46,13 @@ inline float Remap(float x, float lo, float hi, float scale) {
   return (x - lo) / (hi - lo + 1e-6) * scale;
 }
 
-template <class LandmarkType>
-inline void GetMinMaxZ(const std::vector<LandmarkType>& landmarks, float* z_min,
+template <class LandmarkListType, class LandmarkType>
+inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
                        float* z_max) {
   *z_min = std::numeric_limits<float>::max();
   *z_max = std::numeric_limits<float>::min();
-  for (const auto& landmark : landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const LandmarkType& landmark = landmarks.landmark(i);
     *z_min = std::min(landmark.z(), *z_min);
     *z_max = std::max(landmark.z(), *z_max);
   }
@@ -73,7 +74,7 @@ void SetColorSizeValueFromZ(float z, float z_min, float z_max,
 }  // namespace
 
 // A calculator that converts Landmark proto to RenderData proto for
-// visualization. The input should be std::vector<Landmark>. It is also possible
+// visualization. The input should be LandmarkList proto. It is also possible
 // to specify the connections between landmarks.
 //
 // Example config:
@@ -121,11 +122,11 @@ class LandmarksToRenderDataCalculator : public CalculatorBase {
       const LandmarksToRenderDataCalculatorOptions& options, bool normalized,
       int gray_val1, int gray_val2, RenderData* render_data);
 
-  template <class LandmarkType>
-  void AddConnections(const std::vector<LandmarkType>& landmarks,
-                      bool normalized, RenderData* render_data);
-  template <class LandmarkType>
-  void AddConnectionsWithDepth(const std::vector<LandmarkType>& landmarks,
+  template <class LandmarkListType>
+  void AddConnections(const LandmarkListType& landmarks, bool normalized,
+                      RenderData* render_data);
+  template <class LandmarkListType>
+  void AddConnectionsWithDepth(const LandmarkListType& landmarks,
                                bool normalized, float min_z, float max_z,
                                RenderData* render_data);
 
@@ -144,10 +145,10 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
          "normalized landmarks.";
 
   if (cc->Inputs().HasTag(kLandmarksTag)) {
-    cc->Inputs().Tag(kLandmarksTag).Set<std::vector<Landmark>>();
+    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
   }
   if (cc->Inputs().HasTag(kNormLandmarksTag)) {
-    cc->Inputs().Tag(kNormLandmarksTag).Set<std::vector<NormalizedLandmark>>();
+    cc->Inputs().Tag(kNormLandmarksTag).Set<NormalizedLandmarkList>();
   }
   cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
   return ::mediapipe::OkStatus();
@@ -169,16 +170,17 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
   float z_max = 0.f;
 
   if (cc->Inputs().HasTag(kLandmarksTag)) {
-    const auto& landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<std::vector<Landmark>>();
+    const LandmarkList& landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
     RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
         << "Number of entries in landmark connections must be a multiple of 2";
     if (visualize_depth) {
-      GetMinMaxZ(landmarks, &z_min, &z_max);
+      GetMinMaxZ<LandmarkList, Landmark>(landmarks, &z_min, &z_max);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
-    for (const auto& landmark : landmarks) {
+    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+      const Landmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render =
           AddPointRenderData(options_, render_data.get());
       if (visualize_depth) {
@@ -191,25 +193,27 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_y(landmark.y());
     }
     if (visualize_depth) {
-      AddConnectionsWithDepth(landmarks, /*normalized=*/false, z_min, z_max,
-                              render_data.get());
+      AddConnectionsWithDepth<LandmarkList>(landmarks, /*normalized=*/false,
+                                            z_min, z_max, render_data.get());
     } else {
-      AddConnections(landmarks, /*normalized=*/false, render_data.get());
+      AddConnections<LandmarkList>(landmarks, /*normalized=*/false,
+                                   render_data.get());
     }
   }
 
   if (cc->Inputs().HasTag(kNormLandmarksTag)) {
-    const auto& landmarks = cc->Inputs()
-                                .Tag(kNormLandmarksTag)
-                                .Get<std::vector<NormalizedLandmark>>();
+    const NormalizedLandmarkList& landmarks =
+        cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
     RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
         << "Number of entries in landmark connections must be a multiple of 2";
     if (visualize_depth) {
-      GetMinMaxZ(landmarks, &z_min, &z_max);
+      GetMinMaxZ<NormalizedLandmarkList, NormalizedLandmark>(landmarks, &z_min,
+                                                             &z_max);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
-    for (const auto& landmark : landmarks) {
+    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+      const NormalizedLandmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render =
           AddPointRenderData(options_, render_data.get());
       if (visualize_depth) {
@@ -222,10 +226,11 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_y(landmark.y());
     }
     if (visualize_depth) {
-      AddConnectionsWithDepth(landmarks, /*normalized=*/true, z_min, z_max,
-                              render_data.get());
+      AddConnectionsWithDepth<NormalizedLandmarkList>(
+          landmarks, /*normalized=*/true, z_min, z_max, render_data.get());
     } else {
-      AddConnections(landmarks, /*normalized=*/true, render_data.get());
+      AddConnections<NormalizedLandmarkList>(landmarks, /*normalized=*/true,
+                                             render_data.get());
     }
   }
 
@@ -235,13 +240,13 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
   return ::mediapipe::OkStatus();
 }
 
-template <class LandmarkType>
+template <class LandmarkListType>
 void LandmarksToRenderDataCalculator::AddConnectionsWithDepth(
-    const std::vector<LandmarkType>& landmarks, bool normalized, float min_z,
+    const LandmarkListType& landmarks, bool normalized, float min_z,
     float max_z, RenderData* render_data) {
   for (int i = 0; i < options_.landmark_connections_size(); i += 2) {
-    const auto& ld0 = landmarks[options_.landmark_connections(i)];
-    const auto& ld1 = landmarks[options_.landmark_connections(i + 1)];
+    const auto& ld0 = landmarks.landmark(options_.landmark_connections(i));
+    const auto& ld1 = landmarks.landmark(options_.landmark_connections(i + 1));
     const int gray_val1 =
         255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
     const int gray_val2 =
@@ -272,13 +277,13 @@ void LandmarksToRenderDataCalculator::AddConnectionToRenderData(
   connection_annotation->set_thickness(options.thickness());
 }
 
-template <class LandmarkType>
+template <class LandmarkListType>
 void LandmarksToRenderDataCalculator::AddConnections(
-    const std::vector<LandmarkType>& landmarks, bool normalized,
+    const LandmarkListType& landmarks, bool normalized,
     RenderData* render_data) {
   for (int i = 0; i < options_.landmark_connections_size(); i += 2) {
-    const auto& ld0 = landmarks[options_.landmark_connections(i)];
-    const auto& ld1 = landmarks[options_.landmark_connections(i + 1)];
+    const auto& ld0 = landmarks.landmark(options_.landmark_connections(i));
+    const auto& ld1 = landmarks.landmark(options_.landmark_connections(i + 1));
     AddConnectionToRenderData(ld0.x(), ld0.y(), ld1.x(), ld1.y(), options_,
                               normalized, render_data);
   }

mediapipe/calculators/util/top_k_scores_calculator.cc

@@ -29,8 +29,7 @@
 #include "mediapipe/framework/port/statusor.h"
 #include "mediapipe/util/resource_util.h"
 
-#if defined(MEDIAPIPE_LITE) || defined(__EMSCRIPTEN__) || \
-    defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if defined(MEDIAPIPE_MOBILE)
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/helpers.h"
 #else

mediapipe/calculators/video/BUILD

@@ -37,6 +37,67 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "motion_analysis_calculator_proto",
+    srcs = ["motion_analysis_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:motion_analysis_proto",
+    ],
+)
+
+proto_library(
+    name = "flow_packager_calculator_proto",
+    srcs = ["flow_packager_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:flow_packager_proto",
+    ],
+)
+
+proto_library(
+    name = "box_tracker_calculator_proto",
+    srcs = ["box_tracker_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:box_tracker_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "motion_analysis_calculator_cc_proto",
+    srcs = ["motion_analysis_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:motion_analysis_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":motion_analysis_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "flow_packager_calculator_cc_proto",
+    srcs = ["flow_packager_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:flow_packager_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":flow_packager_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "box_tracker_calculator_cc_proto",
+    srcs = ["box_tracker_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":box_tracker_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "flow_to_image_calculator_cc_proto",
     srcs = ["flow_to_image_calculator.proto"],
@@ -131,6 +192,94 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "motion_analysis_calculator",
+    srcs = ["motion_analysis_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":motion_analysis_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:camera_motion",
+        "//mediapipe/util/tracking:camera_motion_cc_proto",
+        "//mediapipe/util/tracking:frame_selection_cc_proto",
+        "//mediapipe/util/tracking:motion_analysis",
+        "//mediapipe/util/tracking:motion_estimation",
+        "//mediapipe/util/tracking:motion_models",
+        "//mediapipe/util/tracking:region_flow_cc_proto",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "flow_packager_calculator",
+    srcs = ["flow_packager_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":flow_packager_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/util/tracking:camera_motion_cc_proto",
+        "//mediapipe/util/tracking:flow_packager",
+        "//mediapipe/util/tracking:region_flow_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "box_tracker_calculator",
+    srcs = ["box_tracker_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":box_tracker_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",  # fixdeps: keep -- required for exobazel build.
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:options_util",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "tracked_detection_manager_calculator",
+    srcs = ["tracked_detection_manager_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:tracked_detection",
+        "//mediapipe/util/tracking:tracked_detection_manager",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+        "@com_google_absl//absl/container:node_hash_map",
+    ],
+    alwayslink = 1,
+)
+
 filegroup(
     name = "test_videos",
     srcs = [
@@ -201,3 +350,64 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
     ],
 )
+
+MEDIAPIPE_DEPS = [
+    "//mediapipe/calculators/video:box_tracker_calculator",
+    "//mediapipe/calculators/video:flow_packager_calculator",
+    "//mediapipe/calculators/video:motion_analysis_calculator",
+    "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
+    "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+]
+
+mediapipe_binary_graph(
+    name = "parallel_tracker_binarypb",
+    graph = "testdata/parallel_tracker_graph.pbtxt",
+    output_name = "testdata/parallel_tracker.binarypb",
+    visibility = ["//visibility:public"],
+    deps = MEDIAPIPE_DEPS,
+)
+
+mediapipe_binary_graph(
+    name = "tracker_binarypb",
+    graph = "testdata/tracker_graph.pbtxt",
+    output_name = "testdata/tracker.binarypb",
+    visibility = ["//visibility:public"],
+    deps = MEDIAPIPE_DEPS,
+)
+
+cc_test(
+    name = "tracking_graph_test",
+    size = "small",
+    srcs = ["tracking_graph_test.cc"],
+    copts = ["-DPARALLEL_INVOKER_ACTIVE"] + select({
+        "//mediapipe:apple": [],
+        "//mediapipe:android": [],
+        "//conditions:default": [],
+    }),
+    data = [
+        ":testdata/lenna.png",
+        ":testdata/parallel_tracker.binarypb",
+        ":testdata/tracker.binarypb",
+    ],
+    deps = [
+        ":box_tracker_calculator",
+        ":box_tracker_calculator_cc_proto",
+        ":flow_packager_calculator",
+        ":motion_analysis_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
+        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:tracking_cc_proto",
+    ],
+)

mediapipe/calculators/video/box_tracker_calculator.proto

@@ -0,0 +1,55 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/box_tracker.proto";
+
+message BoxTrackerCalculatorOptions {
+  extend CalculatorOptions {
+    optional BoxTrackerCalculatorOptions ext = 268767860;
+  }
+
+  optional BoxTrackerOptions tracker_options = 1;
+
+  // Initial position to be tracked. Can also be supplied as side packet or
+  // as input stream.
+  optional TimedBoxProtoList initial_position = 2;
+
+  // If set and VIZ stream is present, renders tracking data into the
+  // visualization.
+  optional bool visualize_tracking_data = 3 [default = false];
+
+  // If set and VIZ stream is present, renders the box state
+  // into the visualization.
+  optional bool visualize_state = 4 [default = false];
+
+  // If set and VIZ stream is present, renders the internal box state
+  // into the visualization.
+  optional bool visualize_internal_state = 5 [default = false];
+
+  // Size of the track data cache during streaming mode. This allows to buffer
+  // track_data's for fast forward tracking, i.e. any TimedBox received
+  // via input stream START_POS can be tracked towards the current track head
+  // (i.e. last received TrackingData). Measured in number of frames.
+  optional int32 streaming_track_data_cache_size = 6 [default = 0];
+
+  // Add a transition period of N frames to smooth the jump from original
+  // tracking to reset start pos with motion compensation. The transition will
+  // be a linear decay of original tracking result. 0 means no transition.
+  optional int32 start_pos_transition_frames = 7 [default = 0];
+}

mediapipe/calculators/video/flow_packager_calculator.cc

@@ -0,0 +1,281 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <fstream>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/calculators/video/flow_packager_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/util/tracking/camera_motion.pb.h"
+#include "mediapipe/util/tracking/flow_packager.h"
+#include "mediapipe/util/tracking/region_flow.pb.h"
+
+namespace mediapipe {
+
+using mediapipe::CameraMotion;
+using mediapipe::FlowPackager;
+using mediapipe::RegionFlowFeatureList;
+using mediapipe::TrackingData;
+using mediapipe::TrackingDataChunk;
+
+// A calculator that packages input CameraMotion and RegionFlowFeatureList
+// into a TrackingData and optionally writes TrackingDataChunks to file.
+//
+// Input stream:
+//   FLOW:       Input region flow (proto RegionFlowFeatureList).
+//   CAMERA:     Input camera stream (proto CameraMotion, optional).
+//
+// Input side packets:
+//   CACHE_DIR:  Optional caching directory tracking files are written to.
+//
+// Output streams.
+//   TRACKING:       Output tracking data (proto TrackingData, per frame
+//                   optional).
+//   TRACKING_CHUNK: Output tracking chunks (proto TrackingDataChunk,
+//                   per chunk, optional), output at the first timestamp
+//                   of each chunk.
+//   COMPLETE:       Optional output packet sent on PreStream to
+//                   to signal downstream calculators that all data has been
+//                   processed and calculator is closed. Can be used to indicate
+//                   that all data as been written to CACHE_DIR.
+class FlowPackagerCalculator : public CalculatorBase {
+ public:
+  ~FlowPackagerCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+  // Writes passed chunk to disk.
+  void WriteChunk(const TrackingDataChunk& chunk) const;
+
+  // Initializes next chunk for tracking beginning from last frame of
+  // current chunk (Chunking is design with one frame overlap).
+  void PrepareCurrentForNextChunk(TrackingDataChunk* chunk);
+
+ private:
+  FlowPackagerCalculatorOptions options_;
+
+  // Caching options.
+  bool use_caching_ = false;
+  bool build_chunk_ = false;
+  std::string cache_dir_;
+  int chunk_idx_ = -1;
+  TrackingDataChunk tracking_chunk_;
+
+  int frame_idx_ = 0;
+
+  Timestamp prev_timestamp_;
+  std::unique_ptr<FlowPackager> flow_packager_;
+};
+
+REGISTER_CALCULATOR(FlowPackagerCalculator);
+
+::mediapipe::Status FlowPackagerCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (!cc->Inputs().HasTag("FLOW")) {
+    return tool::StatusFail("No input flow was specified.");
+  }
+
+  cc->Inputs().Tag("FLOW").Set<RegionFlowFeatureList>();
+
+  if (cc->Inputs().HasTag("CAMERA")) {
+    cc->Inputs().Tag("CAMERA").Set<CameraMotion>();
+  }
+  if (cc->Outputs().HasTag("TRACKING")) {
+    cc->Outputs().Tag("TRACKING").Set<TrackingData>();
+  }
+  if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+    cc->Outputs().Tag("TRACKING_CHUNK").Set<TrackingDataChunk>();
+  }
+  if (cc->Outputs().HasTag("COMPLETE")) {
+    cc->Outputs().Tag("COMPLETE").Set<bool>();
+  }
+
+  if (cc->InputSidePackets().HasTag("CACHE_DIR")) {
+    cc->InputSidePackets().Tag("CACHE_DIR").Set<std::string>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<FlowPackagerCalculatorOptions>();
+
+  flow_packager_.reset(new FlowPackager(options_.flow_packager_options()));
+
+  use_caching_ = cc->InputSidePackets().HasTag("CACHE_DIR");
+  build_chunk_ = use_caching_ || cc->Outputs().HasTag("TRACKING_CHUNK");
+  if (use_caching_) {
+    cache_dir_ = cc->InputSidePackets().Tag("CACHE_DIR").Get<std::string>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Process(CalculatorContext* cc) {
+  InputStream* flow_stream = &(cc->Inputs().Tag("FLOW"));
+  const RegionFlowFeatureList& flow = flow_stream->Get<RegionFlowFeatureList>();
+
+  const Timestamp timestamp = flow_stream->Value().Timestamp();
+
+  const CameraMotion* camera_motion = nullptr;
+  if (cc->Inputs().HasTag("CAMERA")) {
+    InputStream* camera_stream = &(cc->Inputs().Tag("CAMERA"));
+    camera_motion = &camera_stream->Get<CameraMotion>();
+  }
+
+  std::unique_ptr<TrackingData> tracking_data(new TrackingData());
+
+  flow_packager_->PackFlow(flow, camera_motion, tracking_data.get());
+
+  if (build_chunk_) {
+    if (chunk_idx_ < 0) {  // Lazy init, determine first start.
+      chunk_idx_ =
+          timestamp.Value() / 1000 / options_.caching_chunk_size_msec();
+      tracking_chunk_.set_first_chunk(true);
+    }
+    CHECK_GE(chunk_idx_, 0);
+
+    TrackingDataChunk::Item* item = tracking_chunk_.add_item();
+    item->set_frame_idx(frame_idx_);
+    item->set_timestamp_usec(timestamp.Value());
+    if (frame_idx_ > 0) {
+      item->set_prev_timestamp_usec(prev_timestamp_.Value());
+    }
+    if (cc->Outputs().HasTag("TRACKING")) {
+      // Need to copy as output is requested.
+      *item->mutable_tracking_data() = *tracking_data;
+    } else {
+      item->mutable_tracking_data()->Swap(tracking_data.get());
+    }
+
+    const int next_chunk_msec =
+        options_.caching_chunk_size_msec() * (chunk_idx_ + 1);
+
+    if (timestamp.Value() / 1000 >= next_chunk_msec) {
+      if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+        cc->Outputs()
+            .Tag("TRACKING_CHUNK")
+            .Add(new TrackingDataChunk(tracking_chunk_),
+                 Timestamp(tracking_chunk_.item(0).timestamp_usec()));
+      }
+      if (use_caching_) {
+        WriteChunk(tracking_chunk_);
+      }
+      PrepareCurrentForNextChunk(&tracking_chunk_);
+    }
+  }
+
+  if (cc->Outputs().HasTag("TRACKING")) {
+    cc->Outputs()
+        .Tag("TRACKING")
+        .Add(tracking_data.release(), flow_stream->Value().Timestamp());
+  }
+
+  prev_timestamp_ = timestamp;
+  ++frame_idx_;
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Close(CalculatorContext* cc) {
+  if (frame_idx_ > 0) {
+    tracking_chunk_.set_last_chunk(true);
+    if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+      cc->Outputs()
+          .Tag("TRACKING_CHUNK")
+          .Add(new TrackingDataChunk(tracking_chunk_),
+               Timestamp(tracking_chunk_.item(0).timestamp_usec()));
+    }
+
+    if (use_caching_) {
+      WriteChunk(tracking_chunk_);
+    }
+  }
+
+  if (cc->Outputs().HasTag("COMPLETE")) {
+    cc->Outputs().Tag("COMPLETE").Add(new bool(true), Timestamp::PreStream());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const {
+  if (chunk.item_size() == 0) {
+    LOG(ERROR) << "Write chunk called with empty tracking data."
+               << "This can only occur if the spacing between frames "
+               << "is larger than the requested chunk size. Try increasing "
+               << "the chunk size";
+    return;
+  }
+
+  auto format_runtime =
+      absl::ParsedFormat<'d'>::New(options_.cache_file_format());
+
+  std::string chunk_file;
+  if (format_runtime) {
+    chunk_file =
+        cache_dir_ + "/" + absl::StrFormat(*format_runtime, chunk_idx_);
+  } else {
+    LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d.";
+    chunk_file = cache_dir_ + "/" + absl::StrFormat("chunk_%04d", chunk_idx_);
+  }
+
+  std::string data;
+  chunk.SerializeToString(&data);
+
+  const char* temp_filename = tempnam(cache_dir_.c_str(), nullptr);
+  std::ofstream out_file(temp_filename);
+  if (!out_file) {
+    LOG(ERROR) << "Could not open " << temp_filename;
+  } else {
+    out_file.write(data.data(), data.size());
+  }
+
+  if (rename(temp_filename, chunk_file.c_str()) != 0) {
+    LOG(ERROR) << "Failed to rename to " << chunk_file;
+  }
+
+  LOG(INFO) << "Wrote chunk : " << chunk_file;
+}
+
+void FlowPackagerCalculator::PrepareCurrentForNextChunk(
+    TrackingDataChunk* chunk) {
+  CHECK(chunk);
+  if (chunk->item_size() == 0) {
+    LOG(ERROR) << "Called with empty chunk. Unexpected.";
+    return;
+  }
+
+  chunk->set_first_chunk(false);
+
+  // Buffer last item for next chunk.
+  TrackingDataChunk::Item last_item;
+  last_item.Swap(chunk->mutable_item(chunk->item_size() - 1));
+
+  chunk->Clear();
+  chunk->add_item()->Swap(&last_item);
+
+  ++chunk_idx_;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/flow_packager_calculator.proto

@@ -0,0 +1,36 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/flow_packager.proto";
+
+message FlowPackagerCalculatorOptions {
+  extend CalculatorOptions {
+    optional FlowPackagerCalculatorOptions ext = 271236147;
+  }
+
+  optional mediapipe.FlowPackagerOptions flow_packager_options = 1;
+
+  // Chunk size for caching files that are written to the externally specified
+  // caching directory. Specified in msec.
+  // Note that each chunk always contains at its end the first frame of the
+  // next chunk (to enable forward tracking across chunk boundaries).
+  optional int32 caching_chunk_size_msec = 2 [default = 2500];
+
+  optional string cache_file_format = 3 [default = "chunk_%04d"];
+}

mediapipe/calculators/video/motion_analysis_calculator.cc

@@ -0,0 +1,988 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <fstream>
+#include <memory>
+
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/calculators/video/motion_analysis_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/camera_motion.h"
+#include "mediapipe/util/tracking/camera_motion.pb.h"
+#include "mediapipe/util/tracking/frame_selection.pb.h"
+#include "mediapipe/util/tracking/motion_analysis.h"
+#include "mediapipe/util/tracking/motion_estimation.h"
+#include "mediapipe/util/tracking/motion_models.h"
+#include "mediapipe/util/tracking/region_flow.pb.h"
+
+namespace mediapipe {
+
+using mediapipe::AffineAdapter;
+using mediapipe::CameraMotion;
+using mediapipe::FrameSelectionResult;
+using mediapipe::Homography;
+using mediapipe::HomographyAdapter;
+using mediapipe::LinearSimilarityModel;
+using mediapipe::MixtureHomography;
+using mediapipe::MixtureRowWeights;
+using mediapipe::MotionAnalysis;
+using mediapipe::ProjectViaFit;
+using mediapipe::RegionFlowComputationOptions;
+using mediapipe::RegionFlowFeatureList;
+using mediapipe::SalientPointFrame;
+using mediapipe::TranslationModel;
+
+const char kOptionsTag[] = "OPTIONS";
+
+// A calculator that performs motion analysis on an incoming video stream.
+//
+// Input streams:  (at least one of them is required).
+//   VIDEO:     The input video stream (ImageFrame, sRGB, sRGBA or GRAY8).
+//   SELECTION: Optional input stream to perform analysis only on selected
+//              frames. If present needs to contain camera motion
+//              and features.
+//
+// Input side packets:
+//   CSV_FILE:  Read motion models as homographies from CSV file. Expected
+//              to be defined in the frame domain (un-normalized).
+//              Should store 9 floats per row.
+//              Specify number of homographies per frames via option
+//              meta_models_per_frame. For values > 1, MixtureHomographies
+//              are created, for value == 1, a single Homography is used.
+//   DOWNSAMPLE: Optionally specify downsampling factor via input side packet
+//               overriding value in the graph settings.
+// Output streams (all are optional).
+//   FLOW:      Sparse feature tracks in form of proto RegionFlowFeatureList.
+//   CAMERA:    Camera motion as proto CameraMotion describing the per frame-
+//              pair motion. Has VideoHeader from input video.
+//   SALIENCY:  Foreground saliency (objects moving different from the
+//              background) as proto SalientPointFrame.
+//   VIZ:       Visualization stream as ImageFrame, sRGB, visualizing
+//              features and saliency (set via
+//              analysis_options().visualization_options())
+//   DENSE_FG:  Dense foreground stream, describing per-pixel foreground-
+//              ness as confidence between 0 (background) and 255
+//              (foreground). Output is ImageFrame (GRAY8).
+//   VIDEO_OUT: Optional output stream when SELECTION is used. Output is input
+//              VIDEO at the selected frames. Required VIDEO to be present.
+//   GRAY_VIDEO_OUT: Optional output stream for downsampled, grayscale video.
+//                   Requires VIDEO to be present and SELECTION to not be used.
+class MotionAnalysisCalculator : public CalculatorBase {
+  // TODO: Activate once leakr approval is ready.
+  // typedef com::google::android::libraries::micro::proto::Data HomographyData;
+
+ public:
+  ~MotionAnalysisCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  // Outputs results to Outputs() if MotionAnalysis buffered sufficient results.
+  // Otherwise no-op. Set flush to true to force output of all buffered data.
+  void OutputMotionAnalyzedFrames(bool flush, CalculatorContext* cc);
+
+  // Lazy init function to be called on Process.
+  ::mediapipe::Status InitOnProcess(InputStream* video_stream,
+                                    InputStream* selection_stream);
+
+  // Parses CSV file contents to homographies.
+  bool ParseModelCSV(const std::string& contents,
+                     std::deque<Homography>* homographies);
+
+  // Turns list of 9-tuple floating values into set of homographies.
+  bool HomographiesFromValues(const std::vector<float>& homog_values,
+                              std::deque<Homography>* homographies);
+
+  // Appends CameraMotions and features from homographies.
+  // Set append_identity to true to add an identity transform to the beginning
+  // of the each list *in addition* to the motions derived from homographies.
+  void AppendCameraMotionsFromHomographies(
+      const std::deque<Homography>& homographies, bool append_identity,
+      std::deque<CameraMotion>* camera_motions,
+      std::deque<RegionFlowFeatureList>* features);
+
+  // Helper function to subtract current metadata motion from features. Used
+  // for hybrid estimation case.
+  void SubtractMetaMotion(const CameraMotion& meta_motion,
+                          RegionFlowFeatureList* features);
+
+  // Inverse of above function to add back meta motion and replace
+  // feature location with originals after estimation.
+  void AddMetaMotion(const CameraMotion& meta_motion,
+                     const RegionFlowFeatureList& meta_features,
+                     RegionFlowFeatureList* features, CameraMotion* motion);
+
+  MotionAnalysisCalculatorOptions options_;
+  int frame_width_ = -1;
+  int frame_height_ = -1;
+  int frame_idx_ = 0;
+
+  // Buffers incoming video frame packets (if visualization output is requested)
+  std::vector<Packet> packet_buffer_;
+
+  // Buffers incoming timestamps until MotionAnalysis is ready to output via
+  // above OutputMotionAnalyzedFrames.
+  std::vector<Timestamp> timestamp_buffer_;
+
+  // Input indicators for each stream.
+  bool selection_input_ = false;
+  bool video_input_ = false;
+
+  // Output indicators for each stream.
+  bool region_flow_feature_output_ = false;
+  bool camera_motion_output_ = false;
+  bool saliency_output_ = false;
+  bool visualize_output_ = false;
+  bool dense_foreground_output_ = false;
+  bool video_output_ = false;
+  bool grayscale_output_ = false;
+  bool csv_file_input_ = false;
+
+  // Inidicates if saliency should be computed.
+  bool with_saliency_ = false;
+
+  // Set if hybrid meta analysis - see proto for details.
+  bool hybrid_meta_analysis_ = false;
+
+  // Concatenated motions for each selected frame. Used in case
+  // hybrid estimation is requested to fallback to valid models.
+  std::deque<CameraMotion> selected_motions_;
+
+  // Normalized homographies from CSV file or metadata.
+  std::deque<Homography> meta_homographies_;
+  std::deque<CameraMotion> meta_motions_;
+  std::deque<RegionFlowFeatureList> meta_features_;
+
+  // Offset into above meta_motions_ and features_ when using
+  // hybrid meta analysis.
+  int hybrid_meta_offset_ = 0;
+
+  std::unique_ptr<MotionAnalysis> motion_analysis_;
+
+  std::unique_ptr<MixtureRowWeights> row_weights_;
+};
+
+REGISTER_CALCULATOR(MotionAnalysisCalculator);
+
+::mediapipe::Status MotionAnalysisCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("VIDEO")) {
+    cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  }
+
+  // Optional input stream from frame selection calculator.
+  if (cc->Inputs().HasTag("SELECTION")) {
+    cc->Inputs().Tag("SELECTION").Set<FrameSelectionResult>();
+  }
+
+  RET_CHECK(cc->Inputs().HasTag("VIDEO") || cc->Inputs().HasTag("SELECTION"))
+      << "Either VIDEO, SELECTION must be specified.";
+
+  if (cc->Outputs().HasTag("FLOW")) {
+    cc->Outputs().Tag("FLOW").Set<RegionFlowFeatureList>();
+  }
+
+  if (cc->Outputs().HasTag("CAMERA")) {
+    cc->Outputs().Tag("CAMERA").Set<CameraMotion>();
+  }
+
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    cc->Outputs().Tag("SALIENCY").Set<SalientPointFrame>();
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    cc->Outputs().Tag("VIZ").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("DENSE_FG")) {
+    cc->Outputs().Tag("DENSE_FG").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("VIDEO_OUT")) {
+    cc->Outputs().Tag("VIDEO_OUT").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("GRAY_VIDEO_OUT")) {
+    // We only output grayscale video if we're actually performing full region-
+    // flow analysis on the video.
+    RET_CHECK(cc->Inputs().HasTag("VIDEO") &&
+              !cc->Inputs().HasTag("SELECTION"));
+    cc->Outputs().Tag("GRAY_VIDEO_OUT").Set<ImageFrame>();
+  }
+
+  if (cc->InputSidePackets().HasTag("CSV_FILE")) {
+    cc->InputSidePackets().Tag("CSV_FILE").Set<std::string>();
+  }
+  if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) {
+    cc->InputSidePackets().Tag("DOWNSAMPLE").Set<float>();
+  }
+
+  if (cc->InputSidePackets().HasTag(kOptionsTag)) {
+    cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Open(CalculatorContext* cc) {
+  options_ =
+      tool::RetrieveOptions(cc->Options<MotionAnalysisCalculatorOptions>(),
+                            cc->InputSidePackets(), kOptionsTag);
+
+  video_input_ = cc->Inputs().HasTag("VIDEO");
+  selection_input_ = cc->Inputs().HasTag("SELECTION");
+  region_flow_feature_output_ = cc->Outputs().HasTag("FLOW");
+  camera_motion_output_ = cc->Outputs().HasTag("CAMERA");
+  saliency_output_ = cc->Outputs().HasTag("SALIENCY");
+  visualize_output_ = cc->Outputs().HasTag("VIZ");
+  dense_foreground_output_ = cc->Outputs().HasTag("DENSE_FG");
+  video_output_ = cc->Outputs().HasTag("VIDEO_OUT");
+  grayscale_output_ = cc->Outputs().HasTag("GRAY_VIDEO_OUT");
+  csv_file_input_ = cc->InputSidePackets().HasTag("CSV_FILE");
+  hybrid_meta_analysis_ = options_.meta_analysis() ==
+                          MotionAnalysisCalculatorOptions::META_ANALYSIS_HYBRID;
+
+  if (video_output_) {
+    RET_CHECK(selection_input_) << "VIDEO_OUT requires SELECTION input";
+  }
+
+  if (selection_input_) {
+    switch (options_.selection_analysis()) {
+      case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION:
+        RET_CHECK(!visualize_output_)
+            << "Visualization not supported for NO_ANALYSIS_USE_SELECTION";
+        RET_CHECK(!dense_foreground_output_)
+            << "Dense foreground not supported for NO_ANALYSIS_USE_SELECTION";
+        RET_CHECK(!saliency_output_)
+            << "Saliency output not supported for NO_ANALYSIS_USE_SELECTION";
+        break;
+
+      case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE:
+      case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED:
+        RET_CHECK(video_input_) << "Need video input for feature tracking.";
+        break;
+
+      case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES:
+        // Nothing to add here.
+        break;
+    }
+  }
+
+  if (visualize_output_ || dense_foreground_output_ || video_output_) {
+    RET_CHECK(video_input_) << "Video input required.";
+  }
+
+  if (csv_file_input_) {
+    RET_CHECK(!selection_input_)
+        << "Can not use selection input with csv input.";
+    if (!hybrid_meta_analysis_) {
+      RET_CHECK(!saliency_output_ && !visualize_output_ &&
+                !dense_foreground_output_ && !grayscale_output_)
+          << "CSV file and meta input only supports flow and camera motion "
+          << "output when using metadata only.";
+    }
+  }
+
+  if (csv_file_input_) {
+    // Read from file and parse.
+    const std::string filename =
+        cc->InputSidePackets().Tag("CSV_FILE").Get<std::string>();
+
+    std::string file_contents;
+    std::ifstream input_file(filename, std::ios::in);
+    input_file.seekg(0, std::ios::end);
+    const int file_length = input_file.tellg();
+    file_contents.resize(file_length);
+    input_file.seekg(0, std::ios::beg);
+    input_file.read(&file_contents[0], file_length);
+    input_file.close();
+
+    RET_CHECK(ParseModelCSV(file_contents, &meta_homographies_))
+        << "Could not parse CSV file";
+  }
+
+  // Get video header from video or selection input if present.
+  const VideoHeader* video_header = nullptr;
+  if (video_input_ && !cc->Inputs().Tag("VIDEO").Header().IsEmpty()) {
+    video_header = &(cc->Inputs().Tag("VIDEO").Header().Get<VideoHeader>());
+  } else if (selection_input_ &&
+             !cc->Inputs().Tag("SELECTION").Header().IsEmpty()) {
+    video_header = &(cc->Inputs().Tag("SELECTION").Header().Get<VideoHeader>());
+  } else {
+    LOG(WARNING) << "No input video header found. Downstream calculators "
+                    "expecting video headers are likely to fail.";
+  }
+
+  with_saliency_ = options_.analysis_options().compute_motion_saliency();
+  // Force computation of saliency if requested as output.
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    with_saliency_ = true;
+    if (!options_.analysis_options().compute_motion_saliency()) {
+      LOG(WARNING) << "Enable saliency computation. Set "
+                   << "compute_motion_saliency to true to silence this "
+                   << "warning.";
+      options_.mutable_analysis_options()->set_compute_motion_saliency(true);
+    }
+  }
+
+  if (options_.bypass_mode()) {
+    cc->SetOffset(TimestampDiff(0));
+  }
+
+  if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) {
+    options_.mutable_analysis_options()
+        ->mutable_flow_options()
+        ->set_downsample_factor(
+            cc->InputSidePackets().Tag("DOWNSAMPLE").Get<float>());
+  }
+
+  // If no video header is provided, just return and initialize on the first
+  // Process() call.
+  if (video_header == nullptr) {
+    return ::mediapipe::OkStatus();
+  }
+
+  ////////////// EARLY RETURN; ONLY HEADER OUTPUT SHOULD GO HERE ///////////////
+
+  if (visualize_output_) {
+    cc->Outputs().Tag("VIZ").SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (video_output_) {
+    cc->Outputs()
+        .Tag("VIDEO_OUT")
+        .SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (cc->Outputs().HasTag("DENSE_FG")) {
+    std::unique_ptr<VideoHeader> foreground_header(
+        new VideoHeader(*video_header));
+    foreground_header->format = ImageFormat::GRAY8;
+    cc->Outputs().Tag("DENSE_FG").SetHeader(Adopt(foreground_header.release()));
+  }
+
+  if (cc->Outputs().HasTag("CAMERA")) {
+    cc->Outputs().Tag("CAMERA").SetHeader(
+        Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    cc->Outputs()
+        .Tag("SALIENCY")
+        .SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) {
+  if (options_.bypass_mode()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  InputStream* video_stream =
+      video_input_ ? &(cc->Inputs().Tag("VIDEO")) : nullptr;
+  InputStream* selection_stream =
+      selection_input_ ? &(cc->Inputs().Tag("SELECTION")) : nullptr;
+
+  // Checked on Open.
+  CHECK(video_stream || selection_stream);
+
+  // Lazy init.
+  if (frame_width_ < 0 || frame_height_ < 0) {
+    MP_RETURN_IF_ERROR(InitOnProcess(video_stream, selection_stream));
+  }
+
+  const Timestamp timestamp = cc->InputTimestamp();
+  if ((csv_file_input_) && !hybrid_meta_analysis_) {
+    if (camera_motion_output_) {
+      RET_CHECK(!meta_motions_.empty()) << "Insufficient metadata.";
+
+      CameraMotion output_motion = meta_motions_.front();
+      meta_motions_.pop_front();
+      output_motion.set_timestamp_usec(timestamp.Value());
+      cc->Outputs().Tag("CAMERA").Add(new CameraMotion(output_motion),
+                                      timestamp);
+    }
+
+    if (region_flow_feature_output_) {
+      RET_CHECK(!meta_features_.empty()) << "Insufficient frames in CSV file";
+      RegionFlowFeatureList output_features = meta_features_.front();
+      meta_features_.pop_front();
+
+      output_features.set_timestamp_usec(timestamp.Value());
+      cc->Outputs().Tag("FLOW").Add(new RegionFlowFeatureList(output_features),
+                                    timestamp);
+    }
+
+    ++frame_idx_;
+    return ::mediapipe::OkStatus();
+  }
+
+  if (motion_analysis_ == nullptr) {
+    // We do not need MotionAnalysis when using just metadata.
+    motion_analysis_.reset(new MotionAnalysis(options_.analysis_options(),
+                                              frame_width_, frame_height_));
+  }
+
+  std::unique_ptr<FrameSelectionResult> frame_selection_result;
+  // Always use frame if selection is not activated.
+  bool use_frame = !selection_input_;
+  if (selection_input_) {
+    CHECK(selection_stream);
+
+    // Fill in timestamps we process.
+    if (!selection_stream->Value().IsEmpty()) {
+      ASSIGN_OR_RETURN(
+          frame_selection_result,
+          selection_stream->Value().ConsumeOrCopy<FrameSelectionResult>());
+      use_frame = true;
+
+      // Make sure both features and camera motion are present.
+      RET_CHECK(frame_selection_result->has_camera_motion() &&
+                frame_selection_result->has_features())
+          << "Frame selection input error at: " << timestamp
+          << " both camera motion and features need to be "
+             "present in FrameSelectionResult. "
+          << frame_selection_result->has_camera_motion() << " , "
+          << frame_selection_result->has_features();
+    }
+  }
+
+  if (selection_input_ && use_frame &&
+      options_.selection_analysis() ==
+          MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION) {
+    // Output concatenated results, nothing to compute here.
+    if (camera_motion_output_) {
+      cc->Outputs().Tag("CAMERA").Add(
+          frame_selection_result->release_camera_motion(), timestamp);
+    }
+    if (region_flow_feature_output_) {
+      cc->Outputs().Tag("FLOW").Add(frame_selection_result->release_features(),
+                                    timestamp);
+    }
+
+    if (video_output_) {
+      cc->Outputs().Tag("VIDEO_OUT").AddPacket(video_stream->Value());
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  if (use_frame) {
+    if (!selection_input_) {
+      const cv::Mat input_view =
+          formats::MatView(&video_stream->Get<ImageFrame>());
+      if (hybrid_meta_analysis_) {
+        // Seed with meta homography.
+        RET_CHECK(hybrid_meta_offset_ < meta_motions_.size())
+            << "Not enough metadata received for hybrid meta analysis";
+        Homography initial_transform =
+            meta_motions_[hybrid_meta_offset_].homography();
+        std::function<void(RegionFlowFeatureList*)> subtract_helper = std::bind(
+            &MotionAnalysisCalculator::SubtractMetaMotion, this,
+            meta_motions_[hybrid_meta_offset_], std::placeholders::_1);
+
+        // Keep original features before modification around.
+        motion_analysis_->AddFrameGeneric(
+            input_view, timestamp.Value(), initial_transform, nullptr, nullptr,
+            &subtract_helper, &meta_features_[hybrid_meta_offset_]);
+        ++hybrid_meta_offset_;
+      } else {
+        motion_analysis_->AddFrame(input_view, timestamp.Value());
+      }
+    } else {
+      selected_motions_.push_back(frame_selection_result->camera_motion());
+      switch (options_.selection_analysis()) {
+        case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION:
+          return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
+                 << "Should not reach this point!";
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES:
+          motion_analysis_->AddFeatures(frame_selection_result->features());
+          break;
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE: {
+          const cv::Mat input_view =
+              formats::MatView(&video_stream->Get<ImageFrame>());
+          motion_analysis_->AddFrame(input_view, timestamp.Value());
+          break;
+        }
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED: {
+          Homography homography;
+          CameraMotionToHomography(frame_selection_result->camera_motion(),
+                                   &homography);
+          const cv::Mat input_view =
+              formats::MatView(&video_stream->Get<ImageFrame>());
+          motion_analysis_->AddFrameGeneric(input_view, timestamp.Value(),
+                                            homography, &homography);
+          break;
+        }
+      }
+    }
+
+    timestamp_buffer_.push_back(timestamp);
+    ++frame_idx_;
+
+    VLOG_EVERY_N(0, 100) << "Analyzed frame " << frame_idx_;
+
+    // Buffer input frames only if visualization is requested.
+    if (visualize_output_ || video_output_) {
+      packet_buffer_.push_back(video_stream->Value());
+    }
+
+    // If requested, output grayscale thumbnails
+    if (grayscale_output_) {
+      cv::Mat grayscale_mat = motion_analysis_->GetGrayscaleFrameFromResults();
+      std::unique_ptr<ImageFrame> grayscale_image(new ImageFrame(
+          ImageFormat::GRAY8, grayscale_mat.cols, grayscale_mat.rows));
+      cv::Mat image_frame_mat = formats::MatView(grayscale_image.get());
+      grayscale_mat.copyTo(image_frame_mat);
+
+      cc->Outputs()
+          .Tag("GRAY_VIDEO_OUT")
+          .Add(grayscale_image.release(), timestamp);
+    }
+
+    // Output other results, if we have any yet.
+    OutputMotionAnalyzedFrames(false, cc);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Close(CalculatorContext* cc) {
+  // Guard against empty videos.
+  if (motion_analysis_) {
+    OutputMotionAnalyzedFrames(true, cc);
+  }
+  if (csv_file_input_) {
+    if (!meta_motions_.empty()) {
+      LOG(ERROR) << "More motions than frames. Unexpected! Remainder: "
+                 << meta_motions_.size();
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+void MotionAnalysisCalculator::OutputMotionAnalyzedFrames(
+    bool flush, CalculatorContext* cc) {
+  std::vector<std::unique_ptr<RegionFlowFeatureList>> features;
+  std::vector<std::unique_ptr<CameraMotion>> camera_motions;
+  std::vector<std::unique_ptr<SalientPointFrame>> saliency;
+
+  const int buffer_size = timestamp_buffer_.size();
+  const int num_results = motion_analysis_->GetResults(
+      flush, &features, &camera_motions, with_saliency_ ? &saliency : nullptr);
+
+  CHECK_LE(num_results, buffer_size);
+
+  if (num_results == 0) {
+    return;
+  }
+
+  for (int k = 0; k < num_results; ++k) {
+    // Region flow features and camera motion for this frame.
+    auto& feature_list = features[k];
+    auto& camera_motion = camera_motions[k];
+    const Timestamp timestamp = timestamp_buffer_[k];
+
+    if (selection_input_ && options_.hybrid_selection_camera()) {
+      if (camera_motion->type() > selected_motions_.front().type()) {
+        // Composited type is more stable.
+        camera_motion->Swap(&selected_motions_.front());
+      }
+      selected_motions_.pop_front();
+    }
+
+    if (hybrid_meta_analysis_) {
+      AddMetaMotion(meta_motions_.front(), meta_features_.front(),
+                    feature_list.get(), camera_motion.get());
+      meta_motions_.pop_front();
+      meta_features_.pop_front();
+    }
+
+    // Video frame for visualization.
+    std::unique_ptr<ImageFrame> visualization_frame;
+    cv::Mat visualization;
+    if (visualize_output_) {
+      // Initialize visualization frame with original frame.
+      visualization_frame.reset(new ImageFrame());
+      visualization_frame->CopyFrom(packet_buffer_[k].Get<ImageFrame>(), 16);
+      visualization = formats::MatView(visualization_frame.get());
+
+      motion_analysis_->RenderResults(
+          *feature_list, *camera_motion,
+          with_saliency_ ? saliency[k].get() : nullptr, &visualization);
+
+      cc->Outputs().Tag("VIZ").Add(visualization_frame.release(), timestamp);
+    }
+
+    // Output dense foreground mask.
+    if (dense_foreground_output_) {
+      std::unique_ptr<ImageFrame> foreground_frame(
+          new ImageFrame(ImageFormat::GRAY8, frame_width_, frame_height_));
+      cv::Mat foreground = formats::MatView(foreground_frame.get());
+      motion_analysis_->ComputeDenseForeground(*feature_list, *camera_motion,
+                                               &foreground);
+      cc->Outputs().Tag("DENSE_FG").Add(foreground_frame.release(), timestamp);
+    }
+
+    // Output flow features if requested.
+    if (region_flow_feature_output_) {
+      cc->Outputs().Tag("FLOW").Add(feature_list.release(), timestamp);
+    }
+
+    // Output camera motion.
+    if (camera_motion_output_) {
+      cc->Outputs().Tag("CAMERA").Add(camera_motion.release(), timestamp);
+    }
+
+    if (video_output_) {
+      cc->Outputs().Tag("VIDEO_OUT").AddPacket(packet_buffer_[k]);
+    }
+
+    // Output saliency.
+    if (saliency_output_) {
+      cc->Outputs().Tag("SALIENCY").Add(saliency[k].release(), timestamp);
+    }
+  }
+
+  if (hybrid_meta_analysis_) {
+    hybrid_meta_offset_ -= num_results;
+    CHECK_GE(hybrid_meta_offset_, 0);
+  }
+
+  timestamp_buffer_.erase(timestamp_buffer_.begin(),
+                          timestamp_buffer_.begin() + num_results);
+
+  if (visualize_output_ || video_output_) {
+    packet_buffer_.erase(packet_buffer_.begin(),
+                         packet_buffer_.begin() + num_results);
+  }
+}
+
+::mediapipe::Status MotionAnalysisCalculator::InitOnProcess(
+    InputStream* video_stream, InputStream* selection_stream) {
+  if (video_stream) {
+    frame_width_ = video_stream->Get<ImageFrame>().Width();
+    frame_height_ = video_stream->Get<ImageFrame>().Height();
+
+    // Ensure image options are set correctly.
+    auto* region_options =
+        options_.mutable_analysis_options()->mutable_flow_options();
+
+    // Use two possible formats to account for different channel orders.
+    RegionFlowComputationOptions::ImageFormat image_format;
+    RegionFlowComputationOptions::ImageFormat image_format2;
+    switch (video_stream->Get<ImageFrame>().Format()) {
+      case ImageFormat::GRAY8:
+        image_format = image_format2 =
+            RegionFlowComputationOptions::FORMAT_GRAYSCALE;
+        break;
+
+      case ImageFormat::SRGB:
+        image_format = RegionFlowComputationOptions::FORMAT_RGB;
+        image_format2 = RegionFlowComputationOptions::FORMAT_BGR;
+        break;
+
+      case ImageFormat::SRGBA:
+        image_format = RegionFlowComputationOptions::FORMAT_RGBA;
+        image_format2 = RegionFlowComputationOptions::FORMAT_BGRA;
+        break;
+
+      default:
+        RET_CHECK(false) << "Unsupported image format.";
+    }
+    if (region_options->image_format() != image_format &&
+        region_options->image_format() != image_format2) {
+      LOG(WARNING) << "Requested image format in RegionFlowComputation "
+                   << "does not match video stream format. Overriding.";
+      region_options->set_image_format(image_format);
+    }
+
+    // Account for downsampling mode INPUT_SIZE. In this case we are handed
+    // already downsampled frames but the resulting CameraMotion should
+    // be computed on higher resolution as specifed by the downsample scale.
+    if (region_options->downsample_mode() ==
+        RegionFlowComputationOptions::DOWNSAMPLE_TO_INPUT_SIZE) {
+      const float scale = region_options->downsample_factor();
+      frame_width_ = static_cast<int>(std::round(frame_width_ * scale));
+      frame_height_ = static_cast<int>(std::round(frame_height_ * scale));
+    }
+  } else if (selection_stream) {
+    const auto& camera_motion =
+        selection_stream->Get<FrameSelectionResult>().camera_motion();
+    frame_width_ = camera_motion.frame_width();
+    frame_height_ = camera_motion.frame_height();
+  } else {
+    LOG(FATAL) << "Either VIDEO or SELECTION stream need to be specified.";
+  }
+
+  // Filled by CSV file parsing.
+  if (!meta_homographies_.empty()) {
+    CHECK(csv_file_input_);
+    AppendCameraMotionsFromHomographies(meta_homographies_,
+                                        true,  // append identity.
+                                        &meta_motions_, &meta_features_);
+    meta_homographies_.clear();
+  }
+
+  // Filter weights before using for hybrid mode.
+  if (hybrid_meta_analysis_) {
+    auto* motion_options =
+        options_.mutable_analysis_options()->mutable_motion_options();
+    motion_options->set_filter_initialized_irls_weights(true);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+bool MotionAnalysisCalculator::ParseModelCSV(
+    const std::string& contents, std::deque<Homography>* homographies) {
+  std::vector<absl::string_view> values =
+      absl::StrSplit(contents, absl::ByAnyChar(",\n"));
+
+  // Trim off any empty lines.
+  while (values.back().empty()) {
+    values.pop_back();
+  }
+
+  // Convert to float.
+  std::vector<float> homog_values;
+  homog_values.reserve(values.size());
+
+  for (const auto& value : values) {
+    double value_64f;
+    if (!absl::SimpleAtod(value, &value_64f)) {
+      LOG(ERROR) << "Not a double, expected!";
+      return false;
+    }
+
+    homog_values.push_back(value_64f);
+  }
+
+  return HomographiesFromValues(homog_values, homographies);
+}
+
+bool MotionAnalysisCalculator::HomographiesFromValues(
+    const std::vector<float>& homog_values,
+    std::deque<Homography>* homographies) {
+  CHECK(homographies);
+
+  // Obvious constants are obvious :D
+  constexpr int kHomographyValues = 9;
+  if (homog_values.size() % kHomographyValues != 0) {
+    LOG(ERROR) << "Contents not a multiple of " << kHomographyValues;
+    return false;
+  }
+
+  for (int k = 0; k < homog_values.size(); k += kHomographyValues) {
+    std::vector<double> h_vals(kHomographyValues);
+    for (int l = 0; l < kHomographyValues; ++l) {
+      h_vals[l] = homog_values[k + l];
+    }
+
+    // Normalize last entry to 1.
+    if (h_vals[kHomographyValues - 1] == 0) {
+      LOG(ERROR) << "Degenerate homography, last entry is zero";
+      return false;
+    }
+
+    const double scale = 1.0f / h_vals[kHomographyValues - 1];
+    for (int l = 0; l < kHomographyValues; ++l) {
+      h_vals[l] *= scale;
+    }
+
+    Homography h = HomographyAdapter::FromDoublePointer(h_vals.data(), false);
+    homographies->push_back(h);
+  }
+
+  if (homographies->size() % options_.meta_models_per_frame() != 0) {
+    LOG(ERROR) << "Total homographies not a multiple of specified models "
+               << "per frame.";
+    return false;
+  }
+
+  return true;
+}
+
+void MotionAnalysisCalculator::SubtractMetaMotion(
+    const CameraMotion& meta_motion, RegionFlowFeatureList* features) {
+  if (meta_motion.mixture_homography().model_size() > 0) {
+    CHECK(row_weights_ != nullptr);
+    RegionFlowFeatureListViaTransform(meta_motion.mixture_homography(),
+                                      features, -1.0f,
+                                      1.0f,  // subtract transformed.
+                                      true,  // replace feature loc.
+                                      row_weights_.get());
+  } else {
+    RegionFlowFeatureListViaTransform(meta_motion.homography(), features, -1.0f,
+                                      1.0f,   // subtract transformed.
+                                      true);  // replace feature loc.
+  }
+
+  // Clamp transformed features to domain and handle outliers.
+  const float domain_diam =
+      hypot(features->frame_width(), features->frame_height());
+  const float motion_mag = meta_motion.average_magnitude();
+  // Same irls fraction as used by MODEL_MIXTURE_HOMOGRAPHY scaling in
+  // MotionEstimation.
+  const float irls_fraction = options_.analysis_options()
+                                  .motion_options()
+                                  .irls_mixture_fraction_scale() *
+                              options_.analysis_options()
+                                  .motion_options()
+                                  .irls_motion_magnitude_fraction();
+  float err_scale = std::max(1.0f, motion_mag * irls_fraction);
+
+  const float max_err =
+      options_.meta_outlier_domain_ratio() * domain_diam * err_scale;
+  const float max_err_sq = max_err * max_err;
+
+  for (auto& feature : *features->mutable_feature()) {
+    feature.set_x(
+        std::max(0.0f, std::min(features->frame_width() - 1.0f, feature.x())));
+    feature.set_y(
+        std::max(0.0f, std::min(features->frame_height() - 1.0f, feature.y())));
+    // Label anything with large residual motion an outlier.
+    if (FeatureFlow(feature).Norm2() > max_err_sq) {
+      feature.set_irls_weight(0.0f);
+    }
+  }
+}
+
+void MotionAnalysisCalculator::AddMetaMotion(
+    const CameraMotion& meta_motion, const RegionFlowFeatureList& meta_features,
+    RegionFlowFeatureList* features, CameraMotion* motion) {
+  // Restore old feature location.
+  CHECK_EQ(meta_features.feature_size(), features->feature_size());
+  for (int k = 0; k < meta_features.feature_size(); ++k) {
+    auto feature = features->mutable_feature(k);
+    const auto& meta_feature = meta_features.feature(k);
+    feature->set_x(meta_feature.x());
+    feature->set_y(meta_feature.y());
+    feature->set_dx(meta_feature.dx());
+    feature->set_dy(meta_feature.dy());
+  }
+
+  // Composite camera motion.
+  *motion = ComposeCameraMotion(*motion, meta_motion);
+  // Restore type from metadata, i.e. do not declare motions as invalid.
+  motion->set_type(meta_motion.type());
+  motion->set_match_frame(-1);
+}
+
+void MotionAnalysisCalculator::AppendCameraMotionsFromHomographies(
+    const std::deque<Homography>& homographies, bool append_identity,
+    std::deque<CameraMotion>* camera_motions,
+    std::deque<RegionFlowFeatureList>* features) {
+  CHECK(camera_motions);
+  CHECK(features);
+
+  CameraMotion identity;
+  identity.set_frame_width(frame_width_);
+  identity.set_frame_height(frame_height_);
+
+  *identity.mutable_translation() = TranslationModel();
+  *identity.mutable_linear_similarity() = LinearSimilarityModel();
+  *identity.mutable_homography() = Homography();
+  identity.set_type(CameraMotion::VALID);
+  identity.set_match_frame(0);
+
+  RegionFlowFeatureList empty_list;
+  empty_list.set_long_tracks(true);
+  empty_list.set_match_frame(-1);
+  empty_list.set_frame_width(frame_width_);
+  empty_list.set_frame_height(frame_height_);
+
+  if (append_identity) {
+    camera_motions->push_back(identity);
+    features->push_back(empty_list);
+  }
+
+  const int models_per_frame = options_.meta_models_per_frame();
+  CHECK_GT(models_per_frame, 0) << "At least one model per frame is needed";
+  CHECK_EQ(0, homographies.size() % models_per_frame);
+  const int num_frames = homographies.size() / models_per_frame;
+
+  // Heuristic sigma, similar to what we use for rolling shutter removal.
+  const float mixture_sigma = 1.0f / models_per_frame;
+
+  if (row_weights_ == nullptr) {
+    row_weights_.reset(new MixtureRowWeights(frame_height_,
+                                             frame_height_ / 10,  // 10% margin
+                                             mixture_sigma * frame_height_,
+                                             1.0f, models_per_frame));
+  }
+
+  for (int f = 0; f < num_frames; ++f) {
+    MixtureHomography mix_homog;
+    const int model_start = f * models_per_frame;
+
+    for (int k = 0; k < models_per_frame; ++k) {
+      const Homography& homog = homographies[model_start + k];
+      *mix_homog.add_model() = ModelInvert(homog);
+    }
+
+    CameraMotion c = identity;
+    c.set_match_frame(-1);
+
+    if (mix_homog.model_size() > 1) {
+      *c.mutable_mixture_homography() = mix_homog;
+      c.set_mixture_row_sigma(mixture_sigma);
+
+      for (int k = 0; k < models_per_frame; ++k) {
+        c.add_mixture_inlier_coverage(1.0f);
+      }
+      *c.add_mixture_homography_spectrum() = mix_homog;
+      c.set_rolling_shutter_motion_index(0);
+
+      *c.mutable_homography() = ProjectViaFit<Homography>(
+          mix_homog, frame_width_, frame_height_, row_weights_.get());
+    } else {
+      // Guaranteed to exist because to check that models_per_frame > 0 above.
+      *c.mutable_homography() = mix_homog.model(0);
+    }
+
+    // Project remaining motions down.
+    *c.mutable_linear_similarity() = ProjectViaFit<LinearSimilarityModel>(
+        c.homography(), frame_width_, frame_height_);
+    *c.mutable_translation() = ProjectViaFit<TranslationModel>(
+        c.homography(), frame_width_, frame_height_);
+
+    c.set_average_magnitude(
+        std::hypot(c.translation().dx(), c.translation().dy()));
+
+    camera_motions->push_back(c);
+    features->push_back(empty_list);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/motion_analysis_calculator.proto

@@ -0,0 +1,111 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/motion_analysis.proto";
+
+// Next tag: 10
+message MotionAnalysisCalculatorOptions {
+  extend CalculatorOptions {
+    optional MotionAnalysisCalculatorOptions ext = 270698255;
+  }
+
+  optional mediapipe.MotionAnalysisOptions analysis_options = 1;
+
+  // Determines how optional input SELECTION (if present) is used to compute
+  // the final camera motion.
+  enum SelectionAnalysis {
+    // Recompute camera motion for selected frame neighbors.
+    ANALYSIS_RECOMPUTE = 1;
+
+    // Use composited camera motion and region flow from SELECTION input. No
+    // tracking or re-computation is performed.
+    // Note that in this case only CAMERA, FLOW and VIDEO_OUT tags are
+    // supported as output.
+    NO_ANALYSIS_USE_SELECTION = 2;
+
+    // Recompute camera motion for selected frame neighbors using
+    // features supplied by SELECTION input. No feature tracking is performed.
+    ANALYSIS_FROM_FEATURES = 3;
+
+    // Recomputes camera motion for selected frame neighbors but seeds
+    // initial transform with camera motion from SELECTION input.
+    ANALYSIS_WITH_SEED = 4;
+  }
+
+  optional SelectionAnalysis selection_analysis = 4
+      [default = ANALYSIS_WITH_SEED];
+
+  // If activated when SELECTION input is activated, will replace the computed
+  // camera motion (for any of the ANALYSIS_* case above) with the one supplied
+  // by the frame selection, in case the frame selection one is more stable.
+  // For example, if recomputed camera motion is unstable but the one from
+  // the selection result is stable, will use the stable result instead.
+  optional bool hybrid_selection_camera = 5 [default = false];
+
+  // Determines how optional input META is used to compute the final camera
+  // motion.
+  enum MetaAnalysis {
+    // Uses metadata supplied motions as is.
+    META_ANALYSIS_USE_META = 1;
+
+    // Seeds visual tracking from metadata motions - estimates visual residual
+    // motion and combines with metadata.
+    META_ANALYSIS_HYBRID = 2;
+  }
+
+  optional MetaAnalysis meta_analysis = 8 [default = META_ANALYSIS_USE_META];
+
+  // Determines number of homography models per frame stored in the CSV file
+  // or the homography metadata in META.
+  // For values > 1, MixtureHomographies are created.
+  optional int32 meta_models_per_frame = 6 [default = 1];
+
+  // Used for META_ANALYSIS_HYBRID. Rejects features which flow deviates
+  // domain_ratio * image diagonal size from the ground truth metadata motion.
+  optional float meta_outlier_domain_ratio = 9 [default = 0.0015];
+
+  // If true, the MotionAnalysisCalculator will skip all processing and emit no
+  // packets on any output. This is useful for quickly creating different
+  // versions of a MediaPipe graph without changing its structure, assuming that
+  // downstream calculators can handle missing input packets.
+  // TODO: Remove this hack. See b/36485206 for more details.
+  optional bool bypass_mode = 7 [default = false];
+}
+
+// Taken from
+// java/com/google/android/libraries/microvideo/proto/microvideo.proto to
+// satisfy leakr requirements
+// TODO: Remove and use above proto.
+message HomographyData {
+  // For each frame, there are 12 homography matrices stored. Each matrix is
+  // 3x3 (9 elements). This field will contain 12 x 3 x 3  float values. The
+  // first row of the first homography matrix will be followed by the second row
+  // of the first homography matrix, followed by third row of first homography
+  // matrix, followed by the first row of the second homography matrix, etc.
+  repeated float motion_homography_data = 1 [packed = true];
+
+  // Vector containing histogram counts for individual patches in the frame.
+  repeated uint32 histogram_count_data = 2 [packed = true];
+
+  // The width of the frame at the time metadata was sampled.
+  optional int32 frame_width = 3;
+
+  // The height of the frame at the time metadata was sampled.
+  optional int32 frame_height = 4;
+}

mediapipe/calculators/video/opencv_video_decoder_calculator.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <stdlib.h>
+
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -66,6 +68,20 @@ ImageFormat::Format GetImageFormat(int num_channels) {
 //   output_stream: "VIDEO:video_frames"
 //   output_stream: "VIDEO_PRESTREAM:video_header"
 // }
+//
+// OpenCV's VideoCapture doesn't decode audio tracks. If the audio tracks need
+// to be saved, specify an output side packet with tag "SAVED_AUDIO_PATH".
+// The calculator will call FFmpeg binary to save audio tracks as an aac file.
+//
+// Example config:
+// node {
+//   calculator: "OpenCvVideoDecoderCalculator"
+//   input_side_packet: "INPUT_FILE_PATH:input_file_path"
+//   output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+//   output_stream: "VIDEO:video_frames"
+//   output_stream: "VIDEO_PRESTREAM:video_header"
+// }
+//
 class OpenCvVideoDecoderCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -74,6 +90,9 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
     if (cc->Outputs().HasTag("VIDEO_PRESTREAM")) {
       cc->Outputs().Tag("VIDEO_PRESTREAM").Set<VideoHeader>();
     }
+    if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
+      cc->OutputSidePackets().Tag("SAVED_AUDIO_PATH").Set<std::string>();
+    }
     return ::mediapipe::OkStatus();
   }
 
@@ -127,6 +146,25 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
     }
     // Rewind to the very first frame.
     cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0);
+
+    if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
+#ifdef HAVE_FFMPEG
+      std::string saved_audio_path = std::tmpnam(nullptr);
+      system(absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
+                          " -vn -f adts ", saved_audio_path)
+                 .c_str());
+      cc->OutputSidePackets()
+          .Tag("SAVED_AUDIO_PATH")
+          .Set(MakePacket<std::string>(saved_audio_path));
+
+#else
+      return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+             << "OpenCVVideoDecoderCalculator can't save the audio file "
+                "because FFmpeg is not installed. Please remove "
+                "output_side_packet: \"SAVED_AUDIO_PATH\" from the node "
+                "config.";
+#endif
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/video/opencv_video_encoder_calculator.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <stdlib.h>
+
 #include <memory>
 #include <string>
 #include <vector>
@@ -39,8 +41,7 @@ namespace mediapipe {
 // packet. Currently, the calculator only supports one video stream (in
 // mediapipe::ImageFrame).
 //
-// Example config to generate the output video file:
-//
+// Example config:
 // node {
 //   calculator: "OpenCvVideoEncoderCalculator"
 //   input_stream: "VIDEO:video"
@@ -53,6 +54,26 @@ namespace mediapipe {
 //     }
 //   }
 // }
+//
+// OpenCV's VideoWriter doesn't encode audio. If an input side packet with tag
+// "AUDIO_FILE_PATH" is specified, the calculator will call FFmpeg binary to
+// attach the audio file to the video as the last step in Close().
+//
+// Example config:
+// node {
+//   calculator: "OpenCvVideoEncoderCalculator"
+//   input_stream: "VIDEO:video"
+//   input_stream: "VIDEO_PRESTREAM:video_header"
+//   input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
+//   input_side_packet: "AUDIO_FILE_PATH:audio_path"
+//   node_options {
+//     [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+//        codec: "avc1"
+//        video_format: "mp4"
+//     }
+//   }
+// }
+//
 class OpenCvVideoEncoderCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
@@ -77,6 +98,9 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
   }
   RET_CHECK(cc->InputSidePackets().HasTag("OUTPUT_FILE_PATH"));
   cc->InputSidePackets().Tag("OUTPUT_FILE_PATH").Set<std::string>();
+  if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) {
+    cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Set<std::string>();
+  }
   return ::mediapipe::OkStatus();
 }
 
@@ -155,6 +179,27 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
   if (writer_ && writer_->isOpened()) {
     writer_->release();
   }
+  if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) {
+#ifdef HAVE_FFMPEG
+    const std::string& audio_file_path =
+        cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Get<std::string>();
+    // A temp output file is needed because FFmpeg can't do in-place editing.
+    const std::string temp_file_path = std::tmpnam(nullptr);
+    system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
+                        "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
+                        " -i ", audio_file_path,
+                        "  -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
+                        "&& rm ", temp_file_path)
+               .c_str());
+
+#else
+    return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+           << "OpenCVVideoEncoderCalculator can't attach the audio tracks to "
+              "the video because FFmpeg is not installed. Please remove "
+              "input_side_packet: \"AUDIO_FILE_PATH\" from the node "
+              "config.";
+#endif
+  }
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/video/testdata/parallel_tracker_graph.pbtxt

@@ -0,0 +1,134 @@
+input_stream: "image_cpu_frames"
+input_stream: "start_pos"
+input_stream: "ra_track"
+
+num_threads: 4
+
+node: {
+  calculator: "MotionAnalysisCalculator"
+  input_stream: "VIDEO:image_cpu_frames"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+
+  options: {
+    [mediapipe.MotionAnalysisCalculatorOptions.ext]: {
+      analysis_options: {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+
+        flow_options: {
+          # Maybe move down to 50
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          downsample_mode: DOWNSAMPLE_NONE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options: {
+            max_features: 500
+            adaptive_extraction_levels: 2
+            min_eig_val_settings: {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+
+        motion_options: {
+          label_empty_frames_as_valid: false
+        }
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "FlowPackagerCalculator"
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+
+  options: {
+    [mediapipe.FlowPackagerCalculatorOptions.ext]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "START_POS:start_pos"
+  output_stream: "BOXES:boxes"
+  input_side_packet: "OPTIONS:calculator_options"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+        }
+        sync_set: {
+          tag_index: "START_POS"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE
+          object_similarity_min_contd_inliers: 6
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "RA_TRACK:ra_track"
+  output_stream: "RA_BOXES:ra_boxes"
+  input_side_packet: "OPTIONS:calculator_options"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+        }
+        sync_set: {
+          tag_index: "RA_TRACK"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE
+          object_similarity_min_contd_inliers: 6
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}

mediapipe/calculators/video/testdata/tracker_graph.pbtxt

@@ -0,0 +1,122 @@
+input_stream: "image_cpu_frames"
+input_stream: "start_pos"
+input_stream: "cancel_object_id"
+input_stream: "ra_track"
+input_stream: "restart_pos"
+input_stream: "track_time"
+
+num_threads: 4
+
+node: {
+  calculator: "MotionAnalysisCalculator"
+  options: {
+    [mediapipe.MotionAnalysisCalculatorOptions.ext]: {
+      analysis_options: {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+
+        flow_options: {
+          # Maybe move down to 50
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          # For mobile application, downsample before input into graph
+          # and use DOWNSAMPLE_TO_INPUT_SIZE and specify
+          # downsampling_factor option or DOWNSAMPLE input_side_packet
+          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options: {
+            max_features: 500
+            corner_extraction_method: EXTRACTION_FAST
+            adaptive_extraction_levels: 2
+            min_eig_val_settings: {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+      }
+    }
+  }
+
+  # Drops packets if calculator cannot keep up with the input rate.
+  input_stream_handler: {
+    input_stream_handler: "FixedSizeInputStreamHandler"
+  }
+
+  input_stream: "VIDEO:image_cpu_frames"
+  input_side_packet: "DOWNSAMPLE:analysis_downsample_factor"
+  input_side_packet: "OPTIONS:calculator_options"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+}
+
+node: {
+  calculator: "FlowPackagerCalculator"
+
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+  options: {
+    [mediapipe.FlowPackagerCalculatorOptions.ext]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+
+  input_side_packet: "OPTIONS:calculator_options"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "TRACK_TIME:track_time"
+  input_stream: "START_POS:start_pos"
+  input_stream: "RESTART_POS:restart_pos"
+  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  input_stream: "RA_TRACK:ra_track"
+  output_stream: "BOXES:boxes"
+  output_stream: "RA_BOXES:ra_boxes"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+          tag_index: "TRACK_TIME"
+        }
+        sync_set: {
+          tag_index: "START_POS"
+        }
+        sync_set: {
+          tag_index: "RESTART_POS"
+        }
+        sync_set: {
+          tag_index: "CANCEL_OBJECT_ID"
+        }
+        sync_set: {
+          tag_index: "RA_TRACK"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+          object_similarity_min_contd_inliers: 10
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}

mediapipe/calculators/video/tracked_detection_manager_calculator.cc

@@ -0,0 +1,319 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "absl/container/node_hash_map.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+#include "mediapipe/util/tracking/tracked_detection.h"
+#include "mediapipe/util/tracking/tracked_detection_manager.h"
+#include "mediapipe/util/tracking/tracking.h"
+
+namespace mediapipe {
+namespace {
+
+constexpr int kDetectionUpdateTimeOutMS = 5000;
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionBoxesTag[] = "DETECTION_BOXES";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+constexpr char kTrackingBoxesTag[] = "TRACKING_BOXES";
+constexpr char kCancelObjectIdTag[] = "CANCEL_OBJECT_ID";
+
+// Move |src| to the back of |dst|.
+void MoveIds(std::vector<int>* dst, std::vector<int> src) {
+  dst->insert(dst->end(), std::make_move_iterator(src.begin()),
+              std::make_move_iterator(src.end()));
+}
+
+int64 GetInputTimestampMs(::mediapipe::CalculatorContext* cc) {
+  return cc->InputTimestamp().Microseconds() / 1000;  // 1 ms = 1000 us.
+}
+
+// Converts a Mediapipe Detection Proto to a TrackedDetection class.
+std::unique_ptr<TrackedDetection> GetTrackedDetectionFromDetection(
+    const Detection& detection, int64 timestamp) {
+  std::unique_ptr<TrackedDetection> tracked_detection =
+      absl::make_unique<TrackedDetection>(detection.detection_id(), timestamp);
+  const float top = detection.location_data().relative_bounding_box().ymin();
+  const float bottom =
+      detection.location_data().relative_bounding_box().ymin() +
+      detection.location_data().relative_bounding_box().height();
+  const float left = detection.location_data().relative_bounding_box().xmin();
+  const float right = detection.location_data().relative_bounding_box().xmin() +
+                      detection.location_data().relative_bounding_box().width();
+  NormalizedRect bounding_box;
+  bounding_box.set_x_center((left + right) / 2.f);
+  bounding_box.set_y_center((top + bottom) / 2.f);
+  bounding_box.set_height(bottom - top);
+  bounding_box.set_width(right - left);
+  tracked_detection->set_bounding_box(bounding_box);
+
+  for (int i = 0; i < detection.label_size(); ++i) {
+    tracked_detection->AddLabel(detection.label(i), detection.score(i));
+  }
+  return tracked_detection;
+}
+
+// Converts a TrackedDetection class to a Mediapipe Detection Proto.
+Detection GetAxisAlignedDetectionFromTrackedDetection(
+    const TrackedDetection& tracked_detection) {
+  Detection detection;
+  LocationData* location_data = detection.mutable_location_data();
+
+  auto corners = tracked_detection.GetCorners();
+
+  float x_min = std::numeric_limits<float>::max();
+  float x_max = std::numeric_limits<float>::min();
+  float y_min = std::numeric_limits<float>::max();
+  float y_max = std::numeric_limits<float>::min();
+  for (int i = 0; i < 4; ++i) {
+    x_min = std::min(x_min, corners[i].x());
+    x_max = std::max(x_max, corners[i].x());
+    y_min = std::min(y_min, corners[i].y());
+    y_max = std::max(y_max, corners[i].y());
+  }
+  location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);
+  LocationData::RelativeBoundingBox* relative_bbox =
+      location_data->mutable_relative_bounding_box();
+  relative_bbox->set_xmin(x_min);
+  relative_bbox->set_ymin(y_min);
+  relative_bbox->set_width(x_max - x_min);
+  relative_bbox->set_height(y_max - y_min);
+
+  // Use previous id which is the id the object when it's first detected.
+  if (tracked_detection.previous_id() > 0) {
+    detection.set_detection_id(tracked_detection.previous_id());
+  } else {
+    detection.set_detection_id(tracked_detection.unique_id());
+  }
+  for (const auto& label_and_score : tracked_detection.label_to_score_map()) {
+    detection.add_label(label_and_score.first);
+    detection.add_score(label_and_score.second);
+  }
+  return detection;
+}
+
+}  // namespace
+
+// TrackedDetectionManagerCalculator accepts detections and tracking results at
+// different frame rate for real time tracking of targets.
+// Input:
+//   DETECTIONS: A vector<Detection> of newly detected targets.
+//   TRACKING_BOXES: A TimedBoxProtoList which contains a list of tracked boxes
+//   from previous detections.
+//
+// Output:
+//   CANCEL_OBJECT_ID: Ids of targets that are missing/lost such that it should
+//   be removed from tracking.
+//   DETECTIONS: List of detections that are being tracked.
+//   DETECTION_BOXES: List of bounding boxes of detections that are being
+//   tracked.
+//
+// Usage example:
+// node {
+//   calculator: "TrackedDetectionManagerCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   input_stream: "TRACKING_BOXES:boxes"
+//   output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+//   output_stream: "DETECTIONS:output_detections"
+// }
+class TrackedDetectionManagerCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Adds new list of detections to |waiting_for_update_detections_|.
+  void AddDetectionList(const DetectionList& detection_list,
+                        CalculatorContext* cc);
+  void AddDetections(const std::vector<Detection>& detections,
+                     CalculatorContext* cc);
+
+  // Manages existing and new detections.
+  TrackedDetectionManager tracked_detection_manager_;
+
+  // Set of detections that are not up to date yet. These detections will be
+  // added to the detection manager until they got updated from the box tracker.
+  absl::node_hash_map<int, std::unique_ptr<TrackedDetection>>
+      waiting_for_update_detections_;
+};
+REGISTER_CALCULATOR(TrackedDetectionManagerCalculator);
+
+::mediapipe::Status TrackedDetectionManagerCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+  }
+  if (cc->Inputs().HasTag(kDetectionListTag)) {
+    cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+  }
+  if (cc->Inputs().HasTag(kTrackingBoxesTag)) {
+    cc->Inputs().Tag(kTrackingBoxesTag).Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Outputs().HasTag(kCancelObjectIdTag)) {
+    cc->Outputs().Tag(kCancelObjectIdTag).Set<int>();
+  }
+  if (cc->Outputs().HasTag(kDetectionsTag)) {
+    cc->Outputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+  }
+  if (cc->Outputs().HasTag(kDetectionBoxesTag)) {
+    cc->Outputs().Tag(kDetectionBoxesTag).Set<std::vector<NormalizedRect>>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TrackedDetectionManagerCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag("TRACKING_BOXES")) {
+    if (!cc->Inputs().Tag("TRACKING_BOXES").IsEmpty()) {
+      const TimedBoxProtoList& tracked_boxes =
+          cc->Inputs().Tag("TRACKING_BOXES").Get<TimedBoxProtoList>();
+
+      // Collect all detections that are removed.
+      auto removed_detection_ids = absl::make_unique<std::vector<int>>();
+      for (const TimedBoxProto& tracked_box : tracked_boxes.box()) {
+        NormalizedRect bounding_box;
+        bounding_box.set_x_center((tracked_box.left() + tracked_box.right()) /
+                                  2.f);
+        bounding_box.set_y_center((tracked_box.bottom() + tracked_box.top()) /
+                                  2.f);
+        bounding_box.set_height(tracked_box.bottom() - tracked_box.top());
+        bounding_box.set_width(tracked_box.right() - tracked_box.left());
+        bounding_box.set_rotation(tracked_box.rotation());
+        // First check if this box updates a detection that's waiting for
+        // update from the tracker.
+        auto waiting_for_update_detectoin_ptr =
+            waiting_for_update_detections_.find(tracked_box.id());
+        if (waiting_for_update_detectoin_ptr !=
+            waiting_for_update_detections_.end()) {
+          // Add the detection and remove duplicated detections.
+          auto removed_ids = tracked_detection_manager_.AddDetection(
+              std::move(waiting_for_update_detectoin_ptr->second));
+          MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+          waiting_for_update_detections_.erase(
+              waiting_for_update_detectoin_ptr);
+        }
+        auto removed_ids = tracked_detection_manager_.UpdateDetectionLocation(
+            tracked_box.id(), bounding_box, tracked_box.time_msec());
+        MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+      }
+      // TODO: Should be handled automatically in detection manager.
+      auto removed_ids = tracked_detection_manager_.RemoveObsoleteDetections(
+          GetInputTimestampMs(cc) - kDetectionUpdateTimeOutMS);
+      MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+      // TODO: Should be handled automatically in detection manager.
+      removed_ids = tracked_detection_manager_.RemoveOutOfViewDetections();
+      MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+      if (!removed_detection_ids->empty() &&
+          cc->Outputs().HasTag(kCancelObjectIdTag)) {
+        auto timestamp = cc->InputTimestamp();
+        for (int box_id : *removed_detection_ids) {
+          // The timestamp is incremented (by 1 us) because currently the box
+          // tracker calculator only accepts one cancel object ID for any given
+          // timestamp.
+          cc->Outputs()
+              .Tag(kCancelObjectIdTag)
+              .AddPacket(mediapipe::MakePacket<int>(box_id).At(timestamp++));
+        }
+      }
+
+      // Output detections and corresponding bounding boxes.
+      const auto& all_detections =
+          tracked_detection_manager_.GetAllTrackedDetections();
+      auto output_detections = absl::make_unique<std::vector<Detection>>();
+      auto output_boxes = absl::make_unique<std::vector<NormalizedRect>>();
+
+      for (const auto& detection_ptr : all_detections) {
+        const auto& detection = *detection_ptr.second;
+        // Only output detections that are synced.
+        if (detection.last_updated_timestamp() <
+            cc->InputTimestamp().Microseconds() / 1000) {
+          continue;
+        }
+        output_detections->emplace_back(
+            GetAxisAlignedDetectionFromTrackedDetection(detection));
+        output_boxes->emplace_back(detection.bounding_box());
+      }
+      if (cc->Outputs().HasTag(kDetectionsTag)) {
+        cc->Outputs()
+            .Tag(kDetectionsTag)
+            .Add(output_detections.release(), cc->InputTimestamp());
+      }
+
+      if (cc->Outputs().HasTag(kDetectionBoxesTag)) {
+        cc->Outputs()
+            .Tag(kDetectionBoxesTag)
+            .Add(output_boxes.release(), cc->InputTimestamp());
+      }
+    }
+  }
+
+  if (cc->Inputs().HasTag(kDetectionsTag) &&
+      !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) {
+    const auto detections =
+        cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>();
+    AddDetections(detections, cc);
+  }
+
+  if (cc->Inputs().HasTag(kDetectionListTag) &&
+      !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) {
+    const auto detection_list =
+        cc->Inputs().Tag(kDetectionListTag).Get<DetectionList>();
+    AddDetectionList(detection_list, cc);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void TrackedDetectionManagerCalculator::AddDetectionList(
+    const DetectionList& detection_list, CalculatorContext* cc) {
+  for (const auto& detection : detection_list.detection()) {
+    // Convert from microseconds to milliseconds.
+    std::unique_ptr<TrackedDetection> new_detection =
+        GetTrackedDetectionFromDetection(
+            detection, cc->InputTimestamp().Microseconds() / 1000);
+
+    const int id = new_detection->unique_id();
+    waiting_for_update_detections_[id] = std::move(new_detection);
+  }
+}
+
+void TrackedDetectionManagerCalculator::AddDetections(
+    const std::vector<Detection>& detections, CalculatorContext* cc) {
+  for (const auto& detection : detections) {
+    // Convert from microseconds to milliseconds.
+    std::unique_ptr<TrackedDetection> new_detection =
+        GetTrackedDetectionFromDetection(
+            detection, cc->InputTimestamp().Microseconds() / 1000);
+
+    const int id = new_detection->unique_id();
+    waiting_for_update_detections_[id] = std::move(new_detection);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/tracking_graph_test.cc

@@ -0,0 +1,709 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/calculators/video/box_tracker_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/tracking.pb.h"
+
+#ifdef __APPLE__
+#include <CoreFoundation/CoreFoundation.h>
+#endif  // defined(__APPLE__)
+
+namespace mediapipe {
+namespace {
+using ::testing::FloatNear;
+using ::testing::Test;
+
+std::string GetTestDir() {
+#ifdef __APPLE__
+  char path[1024];
+  CFURLRef bundle_url = CFBundleCopyBundleURL(CFBundleGetMainBundle());
+  CFURLGetFileSystemRepresentation(
+      bundle_url, true, reinterpret_cast<UInt8*>(path), sizeof(path));
+  CFRelease(bundle_url);
+  return ::mediapipe::file::JoinPath(path, "testdata");
+#elif defined(__ANDROID__)
+  char path[1024];
+  getcwd(path, sizeof(path));
+  return ::mediapipe::file::JoinPath(path,
+                                     "mediapipe/calculators/video/testdata");
+#else
+  return ::mediapipe::file::JoinPath(
+      "./",
+      // This should match the path of the output files
+      // of the genrule() that generates test model files.
+      "mediapipe/calculators/video/testdata");
+#endif  // defined(__APPLE__)
+}
+
+bool LoadBinaryTestGraph(const std::string& graph_path,
+                         CalculatorGraphConfig* config) {
+  std::ifstream ifs;
+  ifs.open(graph_path.c_str());
+  proto_ns::io::IstreamInputStream in_stream(&ifs);
+  bool success = config->ParseFromZeroCopyStream(&in_stream);
+  ifs.close();
+  if (!success) {
+    LOG(ERROR) << "could not parse test graph: " << graph_path;
+  }
+  return success;
+}
+
+class TrackingGraphTest : public Test {
+ protected:
+  TrackingGraphTest() {}
+
+  void SetUp() override {
+    test_dir_ = GetTestDir();
+    const auto graph_path = file::JoinPath(test_dir_, "tracker.binarypb");
+    ASSERT_TRUE(LoadBinaryTestGraph(graph_path, &config_));
+
+    original_image_ = cv::imread(file::JoinPath(test_dir_, "lenna.png"));
+    CreateInputFramesFromOriginalImage(kNumImages, kTranslationStep,
+                                       &input_frames_packets_);
+
+    const auto& first_input_img = input_frames_packets_[0].Get<ImageFrame>();
+    const int img_width = first_input_img.Width();
+    const int img_height = first_input_img.Height();
+    translation_step_x_ = kTranslationStep / static_cast<float>(img_width);
+    translation_step_y_ = kTranslationStep / static_cast<float>(img_height);
+
+    // Creat new configure and packet dump vector to store output.
+    mediapipe::CalculatorGraphConfig config_copy = config_;
+    mediapipe::tool::AddVectorSink("boxes", &config_copy, &output_packets_);
+    mediapipe::tool::AddVectorSink("ra_boxes", &config_copy,
+                                   &random_access_results_packets_);
+
+    // Initialize graph
+    MP_ASSERT_OK(graph_.Initialize(config_copy));
+
+    const auto parallel_graph_path =
+        file::JoinPath(test_dir_, "parallel_tracker.binarypb");
+    CalculatorGraphConfig parallel_config;
+    ASSERT_TRUE(LoadBinaryTestGraph(parallel_graph_path, &parallel_config));
+    mediapipe::tool::AddVectorSink("boxes", &parallel_config, &output_packets_);
+    mediapipe::tool::AddVectorSink("ra_boxes", &parallel_config,
+                                   &random_access_results_packets_);
+    MP_ASSERT_OK(parallel_graph_.Initialize(parallel_config));
+  }
+
+  void CreateInputFramesFromOriginalImage(
+      int num_images, int translation_step,
+      std::vector<Packet>* input_frames_packets);
+
+  void TearDown() override {
+    output_packets_.clear();
+    random_access_results_packets_.clear();
+  }
+
+  std::unique_ptr<TimedBoxProtoList> MakeBoxList(
+      const Timestamp& timestamp, const std::vector<bool>& is_quad_tracking,
+      const std::vector<bool>& is_pnp_tracking,
+      const std::vector<bool>& reacquisition) const;
+
+  void RunGraphWithSidePacketsAndInputs(
+      const std::map<std::string, mediapipe::Packet>& side_packets,
+      const mediapipe::Packet& start_pos_packet);
+
+  // Utility functions used to judge if a given quad or box is near to the
+  // groundtruth location at a given frame.
+  // Examine box.reacquisition() field equals to `reacquisition`.
+  // `frame` can be float number to account for inter-frame interpolation.
+  void ExpectBoxAtFrame(const TimedBoxProto& box, float frame,
+                        bool reacquisition);
+
+  // Examine box.aspect_ratio() field equals to `aspect_ratio` if asepct_ratio
+  // is positive.
+  void ExpectQuadAtFrame(const TimedBoxProto& box, float frame,
+                         float aspect_ratio, bool reacquisition);
+
+  // Utility function to judge if two quad are near to each other.
+  void ExpectQuadNear(const TimedBoxProto& box1, const TimedBoxProto& box2);
+
+  std::unique_ptr<TimedBoxProtoList> CreateRandomAccessTrackingBoxList(
+      const std::vector<Timestamp>& start_timestamps,
+      const std::vector<Timestamp>& end_timestamps) const;
+
+  CalculatorGraph graph_;
+  CalculatorGraph parallel_graph_;
+  CalculatorGraphConfig config_;
+  std::string test_dir_;
+  cv::Mat original_image_;
+  std::vector<Packet> input_frames_packets_;
+  std::vector<mediapipe::Packet> output_packets_;
+  std::vector<mediapipe::Packet> random_access_results_packets_;
+  float translation_step_x_;  // normalized translation step in x direction
+  float translation_step_y_;  // normalized translation step in y direction
+  static constexpr float kInitialBoxHalfWidthNormalized = 0.25f;
+  static constexpr float kInitialBoxHalfHeightNormalized = 0.25f;
+  static constexpr float kImageAspectRatio = 1.0f;  // for lenna.png
+  static constexpr float kInitialBoxLeft =
+      0.5f - kInitialBoxHalfWidthNormalized;
+  static constexpr float kInitialBoxRight =
+      0.5f + kInitialBoxHalfWidthNormalized;
+  static constexpr float kInitialBoxTop =
+      0.5f - kInitialBoxHalfHeightNormalized;
+  static constexpr float kInitialBoxBottom =
+      0.5f + kInitialBoxHalfHeightNormalized;
+  static constexpr int kFrameIntervalUs = 30000;
+  static constexpr int kNumImages = 8;
+  // Each image is shifted to the right and bottom by kTranslationStep
+  // pixels compared with the previous image.
+  static constexpr int kTranslationStep = 10;
+  static constexpr float kEqualityTolerance = 3e-4f;
+};
+
+void TrackingGraphTest::ExpectBoxAtFrame(const TimedBoxProto& box, float frame,
+                                         bool reacquisition) {
+  EXPECT_EQ(box.reacquisition(), reacquisition);
+  EXPECT_TRUE(box.has_rotation());
+  EXPECT_THAT(box.rotation(), FloatNear(0, kEqualityTolerance));
+  EXPECT_THAT(box.left(),
+              FloatNear(kInitialBoxLeft - frame * translation_step_x_,
+                        kEqualityTolerance));
+  EXPECT_THAT(box.top(), FloatNear(kInitialBoxTop - frame * translation_step_y_,
+                                   kEqualityTolerance));
+  EXPECT_THAT(box.bottom(),
+              FloatNear(kInitialBoxBottom - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(box.right(),
+              FloatNear(kInitialBoxRight - frame * translation_step_x_,
+                        kEqualityTolerance));
+}
+
+void TrackingGraphTest::ExpectQuadAtFrame(const TimedBoxProto& box, float frame,
+                                          float aspect_ratio,
+                                          bool reacquisition) {
+  EXPECT_TRUE(box.has_quad()) << "quad must exist!";
+  if (aspect_ratio > 0) {
+    EXPECT_TRUE(box.has_aspect_ratio());
+    EXPECT_NEAR(box.aspect_ratio(), aspect_ratio, kEqualityTolerance);
+  }
+
+  EXPECT_EQ(box.reacquisition(), reacquisition);
+
+  const auto& quad = box.quad();
+  EXPECT_EQ(8, quad.vertices_size())
+      << "quad has only " << box.quad().vertices_size() << " vertices";
+  EXPECT_THAT(quad.vertices(0),
+              FloatNear(kInitialBoxLeft - frame * translation_step_x_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(1),
+              FloatNear(kInitialBoxTop - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(3),
+              FloatNear(kInitialBoxBottom - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(4),
+              FloatNear(kInitialBoxRight - frame * translation_step_x_,
+                        kEqualityTolerance));
+}
+
+void TrackingGraphTest::ExpectQuadNear(const TimedBoxProto& box1,
+                                       const TimedBoxProto& box2) {
+  EXPECT_TRUE(box1.has_quad());
+  EXPECT_TRUE(box2.has_quad());
+  EXPECT_EQ(8, box1.quad().vertices_size())
+      << "quad has only " << box1.quad().vertices_size() << " vertices";
+  EXPECT_EQ(8, box2.quad().vertices_size())
+      << "quad has only " << box2.quad().vertices_size() << " vertices";
+  for (int j = 0; j < box1.quad().vertices_size(); ++j) {
+    EXPECT_NEAR(box1.quad().vertices(j), box2.quad().vertices(j),
+                kEqualityTolerance);
+  }
+}
+
+std::unique_ptr<TimedBoxProtoList> TrackingGraphTest::MakeBoxList(
+    const Timestamp& timestamp, const std::vector<bool>& is_quad_tracking,
+    const std::vector<bool>& is_pnp_tracking,
+    const std::vector<bool>& reacquisition) const {
+  auto box_list = absl::make_unique<TimedBoxProtoList>();
+  int box_id = 0;
+  for (int j = 0; j < is_quad_tracking.size(); ++j) {
+    TimedBoxProto* box = box_list->add_box();
+    if (is_quad_tracking[j]) {
+      box->mutable_quad()->add_vertices(kInitialBoxLeft);
+      box->mutable_quad()->add_vertices(kInitialBoxTop);
+      box->mutable_quad()->add_vertices(kInitialBoxLeft);
+      box->mutable_quad()->add_vertices(kInitialBoxBottom);
+      box->mutable_quad()->add_vertices(kInitialBoxRight);
+      box->mutable_quad()->add_vertices(kInitialBoxBottom);
+      box->mutable_quad()->add_vertices(kInitialBoxRight);
+      box->mutable_quad()->add_vertices(kInitialBoxTop);
+
+      if (is_pnp_tracking[j]) {
+        box->set_aspect_ratio(kImageAspectRatio);
+      }
+    } else {
+      box->set_left(kInitialBoxLeft);
+      box->set_right(kInitialBoxRight);
+      box->set_top(kInitialBoxTop);
+      box->set_bottom(kInitialBoxBottom);
+    }
+    box->set_id(box_id++);
+    box->set_time_msec(timestamp.Value() / 1000);
+    box->set_reacquisition(reacquisition[j]);
+  }
+
+  return box_list;
+}
+
+void TrackingGraphTest::CreateInputFramesFromOriginalImage(
+    int num_images, int translation_step,
+    std::vector<Packet>* input_frames_packets) {
+  const int crop_width = original_image_.cols - num_images * translation_step;
+  const int crop_height = original_image_.rows - num_images * translation_step;
+  for (int i = 0; i < num_images; ++i) {
+    cv::Rect roi(i * translation_step, i * translation_step, crop_width,
+                 crop_height);
+    cv::Mat cropped_img = cv::Mat(original_image_, roi);
+    auto cropped_image_frame = absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, crop_width, crop_height, cropped_img.step[0],
+        cropped_img.data, ImageFrame::PixelDataDeleter::kNone);
+    Timestamp curr_timestamp = Timestamp(i * kFrameIntervalUs);
+    Packet image_packet =
+        Adopt(cropped_image_frame.release()).At(curr_timestamp);
+    input_frames_packets->push_back(image_packet);
+  }
+}
+
+void TrackingGraphTest::RunGraphWithSidePacketsAndInputs(
+    const std::map<std::string, mediapipe::Packet>& side_packets,
+    const mediapipe::Packet& start_pos_packet) {
+  // Start running the graph
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet));
+
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("image_cpu_frames", frame_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
+std::unique_ptr<TimedBoxProtoList>
+TrackingGraphTest::CreateRandomAccessTrackingBoxList(
+    const std::vector<Timestamp>& start_timestamps,
+    const std::vector<Timestamp>& end_timestamps) const {
+  CHECK_EQ(start_timestamps.size(), end_timestamps.size());
+  auto ra_boxes = absl::make_unique<TimedBoxProtoList>();
+  for (int i = 0; i < start_timestamps.size(); ++i) {
+    auto start_box_list =
+        MakeBoxList(start_timestamps[i], std::vector<bool>{true},
+                    std::vector<bool>{true}, std::vector<bool>{false});
+    auto end_box_list =
+        MakeBoxList(end_timestamps[i], std::vector<bool>{true},
+                    std::vector<bool>{true}, std::vector<bool>{false});
+    *(ra_boxes->add_box()) = (*start_box_list).box(0);
+    *(ra_boxes->add_box()) = (*end_box_list).box(0);
+  }
+  return ra_boxes;
+}
+
+TEST_F(TrackingGraphTest, BasicBoxTrackingSanityCheck) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(CalculatorOptions())));
+
+  // Run the graph with input side packets, start_pos, and input image frames.
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // is_quad_tracking is used to indicate whether to track quad for each
+  // individual box.
+  std::vector<bool> is_quad_tracking{false};
+  // is_pnp_tracking is used to indicate whether to use perspective transform to
+  // track quad.
+  std::vector<bool> is_pnp_tracking{false};
+  // is_reacquisition is used to indicate whether to enable reacquisition for
+  // the box.
+  std::vector<bool> is_reacquisition{false};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+  RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet);
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    ExpectBoxAtFrame(boxes.box(0), i, false);
+  }
+}
+
+TEST_F(TrackingGraphTest, BasicQuadTrackingSanityCheck) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // Box id 0 use quad tracking with 8DoF homography transform.
+  // Box id 1 use quad tracking with 6DoF perspective transform.
+  // Box id 2 use box tracking with 4DoF similarity transform.
+  std::vector<bool> is_quad_tracking{true, true, false};
+  std::vector<bool> is_pnp_tracking{false, true, false};
+  std::vector<bool> is_reacquisition{true, false, true};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+  RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet);
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    for (int j = 0; j < boxes.box_size(); ++j) {
+      const TimedBoxProto& box = boxes.box(j);
+      if (is_quad_tracking[box.id()]) {
+        ExpectQuadAtFrame(box, i,
+                          is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f,
+                          is_reacquisition[box.id()]);
+      } else {
+        ExpectBoxAtFrame(box, i, is_reacquisition[box.id()]);
+      }
+    }
+  }
+}
+
+TEST_F(TrackingGraphTest, TestRandomAccessTrackingResults) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  ASSERT_GT(input_frames_packets_.size(), 2);  // at least 3 frames
+  ASSERT_TRUE(input_frames_packets_[2].Timestamp() -
+                  input_frames_packets_[1].Timestamp() >
+              TimestampDiff(1000));
+
+  constexpr int start_frame = 0;
+  Timestamp start_box_time = input_frames_packets_[start_frame].Timestamp();
+  auto start_box_list =
+      MakeBoxList(start_box_time, std::vector<bool>{true},
+                  std::vector<bool>{true}, std::vector<bool>{false});
+  constexpr int end_frame = 2;
+  Timestamp end_box_time = input_frames_packets_[end_frame].Timestamp();
+
+  // Also test reverse random access tracking.
+  // This offset of 1ms is simulating the case where the start query timestamp
+  // to be not any existing frame timestamp. In reality, it's highly encouraged
+  // to have the start query timestamp be aligned with frame timestamp.
+  constexpr int reverse_start_frame = 1;
+  Timestamp reverse_start_box_time =
+      input_frames_packets_[reverse_start_frame].Timestamp() + 1000;
+
+  auto ra_boxes = CreateRandomAccessTrackingBoxList(
+      {start_box_time, reverse_start_box_time}, {end_box_time, start_box_time});
+
+  Packet ra_packet = Adopt(ra_boxes.release()).At(start_box_time);
+  Packet start_packet = Adopt(start_box_list.release()).At(start_box_time);
+
+  // Start running the ordinary graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_packet));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("image_cpu_frames", frame_packet));
+    Packet track_time_packet = Adopt(new int(0)).At(frame_packet.Timestamp());
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("track_time", track_time_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  const TimedBoxProtoList tracking_result =
+      output_packets_[end_frame].Get<TimedBoxProtoList>();
+  EXPECT_EQ(1, tracking_result.box_size());
+
+  // Should have 1 random access packet.
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const TimedBoxProtoList& ra_result =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  // Two box tracking results. One for comparison with normal tracking. The
+  // other for reverse random access tracking.
+  EXPECT_EQ(2, ra_result.box_size());
+
+  // Check if randan access tracking has same result with normal tracking.
+  ExpectQuadNear(tracking_result.box(0), ra_result.box(0));
+  ExpectQuadAtFrame(ra_result.box(0), end_frame - start_frame,
+                    kImageAspectRatio, false);
+  ExpectQuadAtFrame(ra_result.box(1), start_frame - reverse_start_frame - 1,
+                    kImageAspectRatio, false);
+
+  // Clear output and ra result packet vector before test parallel graph.
+  TearDown();
+
+  // Start running the parallel graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(parallel_graph_.StartRun(side_packets));
+  MP_EXPECT_OK(
+      parallel_graph_.AddPacketToInputStream("start_pos", start_packet));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames",
+                                                        frame_packet));
+    MP_EXPECT_OK(parallel_graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(parallel_graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  const TimedBoxProtoList parallel_tracking_result =
+      output_packets_[end_frame].Get<TimedBoxProtoList>();
+  EXPECT_EQ(1, parallel_tracking_result.box_size());
+
+  // should have only 1 random access
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const TimedBoxProtoList& parallel_ra_result =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  EXPECT_EQ(2, parallel_ra_result.box_size());
+
+  // Check if randan access tracking has same result with normal tracking.
+  ExpectQuadNear(parallel_tracking_result.box(0), parallel_ra_result.box(0));
+  ExpectQuadAtFrame(parallel_ra_result.box(0), end_frame - start_frame,
+                    kImageAspectRatio, false);
+  ExpectQuadAtFrame(parallel_ra_result.box(1),
+                    start_frame - reverse_start_frame - 1, kImageAspectRatio,
+                    false);
+}
+
+// Tests what happens when random access request timestamps are
+// outside of cache.
+TEST_F(TrackingGraphTest, TestRandomAccessTrackingTimestamps) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  // We intentionally don't cache all frames, to see what happens when
+  // random access tracking request time falls outside cache range.
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->set_streaming_track_data_cache_size(input_frames_packets_.size() - 1);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  // Set up random access boxes
+  const int num_frames = input_frames_packets_.size();
+  const int64 usec_in_sec = 1000000;
+  std::vector<Timestamp> start_timestamps{
+      input_frames_packets_[0].Timestamp() - usec_in_sec,  // forward
+      input_frames_packets_[0].Timestamp(),                // forward
+      input_frames_packets_[1].Timestamp(),                // forward
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec,  // fwd
+      input_frames_packets_[0].Timestamp(),               // backward
+      input_frames_packets_[num_frames - 1].Timestamp(),  // backward
+      input_frames_packets_[num_frames - 1].Timestamp(),  // backward
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec  // back
+  };
+  std::vector<Timestamp> end_timestamps{
+      input_frames_packets_[num_frames - 1].Timestamp(),
+      input_frames_packets_[num_frames - 1].Timestamp(),
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec,
+      input_frames_packets_[num_frames - 1].Timestamp() + 2 * usec_in_sec,
+      input_frames_packets_[0].Timestamp() - usec_in_sec,
+      input_frames_packets_[0].Timestamp(),
+      input_frames_packets_[0].Timestamp() - usec_in_sec,
+      input_frames_packets_[1].Timestamp()};
+  auto ra_boxes =
+      CreateRandomAccessTrackingBoxList(start_timestamps, end_timestamps);
+  Packet ra_packet =
+      Adopt(ra_boxes.release()).At(input_frames_packets_[0].Timestamp());
+
+  // Run the graph and check if the outside-cache request have no results.
+  // Start running the parallel graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(parallel_graph_.StartRun(side_packets));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames",
+                                                        frame_packet));
+    MP_EXPECT_OK(parallel_graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(parallel_graph_.WaitUntilDone());
+
+  // should have 1 random access packet with 0 result boxes
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const auto& ra_returned_boxes =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  const int num_returned_ra_boxes = ra_returned_boxes.box_size();
+  EXPECT_EQ(0, num_returned_ra_boxes);
+}
+
+TEST_F(TrackingGraphTest, TestTransitionFramesForReacquisition) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  constexpr int kTransitionFrames = 3;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->set_start_pos_transition_frames(kTransitionFrames);
+
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // Box id 0 use quad tracking with 8DoF homography transform.
+  // Box id 1 use quad tracking with 6DoF perspective transform.
+  // Box id 2 use box tracking with 4DoF similarity transform.
+  std::vector<bool> is_quad_tracking{true, true, false};
+  std::vector<bool> is_pnp_tracking{false, true, false};
+  std::vector<bool> is_reacquisition{true, true, true};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+
+  // Setting box pos restart from initial position (frame 0's position).
+  constexpr int kRestartFrame = 3;
+  Timestamp restart_box_time = input_frames_packets_[kRestartFrame].Timestamp();
+  auto restart_box_list = MakeBoxList(restart_box_time, is_quad_tracking,
+                                      is_pnp_tracking, is_reacquisition);
+  Packet restart_pos_packet =
+      Adopt(restart_box_list.release()).At(restart_box_time);
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet));
+
+  for (int j = 0; j < input_frames_packets_.size(); ++j) {
+    // Add TRACK_TIME stream queries in between 2 frames.
+    if (j > 0) {
+      Timestamp track_time = Timestamp((j - 0.5f) * kFrameIntervalUs);
+      LOG(INFO) << track_time.Value();
+      Packet track_time_packet = Adopt(new Timestamp).At(track_time);
+      MP_EXPECT_OK(
+          graph_.AddPacketToInputStream("track_time", track_time_packet));
+    }
+
+    MP_EXPECT_OK(graph_.AddPacketToInputStream("image_cpu_frames",
+                                               input_frames_packets_[j]));
+    Packet track_time_packet =
+        Adopt(new int(0)).At(input_frames_packets_[j].Timestamp());
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("track_time", track_time_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+
+    if (j == kRestartFrame) {
+      MP_EXPECT_OK(
+          graph_.AddPacketToInputStream("restart_pos", restart_pos_packet));
+    }
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size() * 2 - 1, output_packets_.size());
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    float frame_id = i / 2.0f;
+    float expected_frame_id;
+    if (frame_id <= kRestartFrame) {
+      // before transition
+      expected_frame_id = frame_id;
+    } else {
+      float transition_frames = frame_id - kRestartFrame;
+      if (transition_frames <= kTransitionFrames) {
+        // transitioning.
+        expected_frame_id =
+            kRestartFrame -
+            transition_frames / kTransitionFrames * kRestartFrame +
+            transition_frames;
+      } else {
+        // after transition.
+        expected_frame_id = transition_frames;
+      }
+    }
+
+    for (int j = 0; j < boxes.box_size(); ++j) {
+      const TimedBoxProto& box = boxes.box(j);
+      if (is_quad_tracking[box.id()]) {
+        ExpectQuadAtFrame(box, expected_frame_id,
+                          is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f,
+                          is_reacquisition[box.id()]);
+      } else {
+        ExpectBoxAtFrame(box, expected_frame_id, is_reacquisition[box.id()]);
+      }
+    }
+  }
+}
+
+// TODO: Add test for reacquisition.
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/docs/android_archive_library.md

@@ -126,6 +126,6 @@ project.
     ```
 
 6.  Follow our Android app examples to use MediaPipe in Android Studio for your
-    use case. If you are looking for an example, a working face detection
+    use case. If you are looking for an example, a face detection
     example can be found
-    [here](https://github.com/jiuqiant/mediapipe_aar_example).
+    [here](https://github.com/jiuqiant/mediapipe_face_detection_aar_example) and a multi-hand tracking example can be found [here](https://github.com/jiuqiant/mediapipe_multi_hands_tracking_aar_example).

mediapipe/docs/examples.md

@@ -39,6 +39,11 @@ pipeline.
 the same TFLite model in a CPU-based pipeline. This example highlights how
 graphs can be easily adapted to run on CPU v.s. GPU.
 
+### Object Detection and Tracking with GPU
+
+[Object Detection and Tracking with GPU](./object_tracking_mobile_gpu.md) illustrates how to
+use MediaPipe for object detection and tracking.
+
 ### Face Detection with GPU
 
 [Face Detection with GPU](./face_detection_mobile_gpu.md) illustrates how to use

mediapipe/docs/gpu.md

@@ -1,10 +1,10 @@
 ## Running on GPUs
 
 -   [Overview](#overview)
--   [OpenGL Support](#graphconfig)
+-   [OpenGL Support](#opengl-support)
 -   [Life of a GPU calculator](#life-of-a-gpu-calculator)
 -   [GpuBuffer to ImageFrame converters](#gpubuffer-to-imageframe-converters)
-
+-   [Disable GPU support](#disable-gpu-support)
 
 ### Overview
 MediaPipe supports calculator nodes for GPU compute and rendering, and allows combining multiple GPU nodes, as well as mixing them with CPU based calculator nodes. There exist several GPU APIs on mobile platforms (eg, OpenGL ES, Metal and Vulkan). MediaPipe does not attempt to offer a single cross-API GPU abstraction. Individual nodes can be written using different APIs, allowing them to take advantage of platform specific features when needed.
@@ -23,6 +23,7 @@ Below are the design principles for GPU support in MediaPipe
    * A calculator should be allowed maximum flexibility in using the GPU for all or part of its operation, combining it with the CPU if necessary.
 
 ### OpenGL support
+
 MediaPipe supports OpenGL ES up to version 3.2 on Android and up to ES 3.0 on iOS. In addition, MediaPipe also supports Metal on iOS.
 
    * MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this can be very useful in graphs that combine a slower GPU inference path (eg, at 10 FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context corresponds to one sequential command queue, using the same context for both tasks would reduce the rendering frame rate. One challenge MediaPipe's use of multiple contexts solves is the ability to communicate across them. An example scenario is one with an input video that is sent to both the rendering and inferences paths, and rendering needs to have access to the latest output from inference.
@@ -128,3 +129,26 @@ The below diagram shows the data flow in a mobile application that captures vide
 |:--:|
 | *Video frames from the camera are fed into the graph as `GpuBuffer` packets. The input stream is accessed by two calculators in parallel. `GpuBufferToImageFrameCalculator` converts the buffer into an `ImageFrame`, which is then sent through a grayscale converter and a canny filter (both based on OpenCV and running on the CPU), whose output is then converted into a `GpuBuffer` again. A multi-input GPU calculator, GlOverlayCalculator, takes as input both the original `GpuBuffer` and the one coming out of the edge detector, and overlays them using a shader. The output is then sent back to the application using a callback calculator, and the application renders the image to the screen using OpenGL.* |
 
+### Disable GPU Support
+
+By default, building MediaPipe (with no special bazel flags) attempts to compile
+and link against OpenGL/Metal libraries.
+
+There are some command line build flags available to disable/enable GPU support
+within the MediaPipe framework:
+
+```
+# To disable *all* gpu support
+bazel build --define MEDIAPIPE_DISABLE_GPU=1  <my-target>
+
+# to enable full GPU support (OpenGL ES 3.1+ & Metal)
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS  <my-target>
+
+# to enable only OpenGL ES 3.0 and below (no GLES 3.1+ features)
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
+```
+
+Note *MEDIAPIPE_DISABLE_GL_COMPUTE* is automatically defined on all Apple
+systems (Apple doesn't support OpenGL ES 3.1+).
+
+Note on iOS and Android, it is assumed that GPU support will be enabled.

mediapipe/docs/install.md

@@ -63,6 +63,7 @@ To build and run iOS apps:
 
     ```bash
     $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+                           libopencv-calib3d-dev libopencv-features2d-dev \
                            libopencv-imgproc-dev libopencv-video-dev
     ```
 
@@ -245,19 +246,23 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-3.  Install Bazel (0.24.1 and above required).
+3.  Install Bazel (version between 0.24.1 and 1.1.0).
 
-    Option 1. Use package manager tool to install the latest version of Bazel.
+    Option 1. Use package manager tool to install Bazel 1.1.0
 
     ```bash
-    $ brew install bazel
-
-    # Run 'bazel version' to check version of bazel installed
+    # If Bazel 1.1.0+ was installed.
+    $ brew uninstall bazel
+    # Install Bazel 1.1.0
+    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb
+    $ brew link bazel
+    # Run 'bazel version' to check version of bazel
     ```
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install any version of Bazel manually.
+    to install any version of Bazel manually. Note that MediaPipe doesn't
+    support Bazel 1.1.0+ on macOS yet.
 
 4.  Install OpenCV and FFmpeg.
 
@@ -375,7 +380,7 @@ To build and run iOS apps:
 4.  Install the needed packages.
 
     ```bash
-    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential git python zip adb openjdk-8-jdk
+    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
 5.  Install Bazel (0.24.1 and above required).
@@ -407,6 +412,7 @@ To build and run iOS apps:
 
     ```bash
     username@DESKTOP-TMVLBJ1:~/mediapipe$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+                           libopencv-calib3d-dev libopencv-features2d-dev \
                            libopencv-imgproc-dev libopencv-video-dev
     ```
 
@@ -526,7 +532,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
     ```bash
     $ docker run -it --name mediapipe mediapipe:latest
 
-    root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk
+    root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk.sh
 
     # Should print:
     # Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b

mediapipe/docs/mediapipe_ios_setup.md

@@ -7,7 +7,8 @@
 
 2.  Install [Bazel](https://bazel.build/).
 
-    See their [instructions](https://docs.bazel.build/versions/master/install-os-x.html).
+    See their
+    [instructions](https://docs.bazel.build/versions/master/install-os-x.html).
     We recommend using [Homebrew](https://brew.sh/):
 
     ```bash
@@ -15,13 +16,23 @@
     brew install bazelbuild/tap/bazel
     ```
 
-3.  Clone the MediaPipe repository.
+3.  Install python "future" and "six".
+
+    To make Mediapipe work with TensorFlow, please install the python "future"
+    library and the python "six" library:
+
+    ```bash
+    pip install --user future six
+    ```
+
+4.  Clone the MediaPipe repository.
 
     ```bash
     git clone https://github.com/google/mediapipe.git
     ```
 
-4.  Symlink or copy your provisioning profile to `mediapipe/mediapipe/provisioning_profile.mobileprovision`.
+5.  Symlink or copy your provisioning profile to
+    `mediapipe/mediapipe/provisioning_profile.mobileprovision`.
 
     ```bash
     cd mediapipe

mediapipe/docs/multi_hand_tracking_mobile_gpu.md

@@ -560,7 +560,7 @@ node {
 # BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
 # timestamp.
 node {
-  calculator: "EndLoopNormalizedLandmarksVectorCalculator"
+  calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
   input_stream: "ITEM:single_hand_landmarks"
   input_stream: "BATCH_END:single_hand_rect_timestamp"
   output_stream: "ITERABLE:multi_hand_landmarks"
@@ -580,7 +580,7 @@ node {
 # hand. If the hand presence for hand #i is false, the set of landmarks
 # corresponding to that hand are dropped from the vector.
 node {
-  calculator: "FilterLandmarksCollectionCalculator"
+  calculator: "FilterLandmarkListCollectionCalculator"
   input_stream: "ITERABLE:multi_hand_landmarks"
   input_stream: "CONDITION:multi_hand_presence"
   output_stream: "ITERABLE:filtered_multi_hand_landmarks"
@@ -669,7 +669,7 @@ node {
 # timestamp for downstream calculators to inform them that all elements in the
 # vector have been processed.
 node {
-  calculator: "BeginLoopNormalizedLandmarksVectorCalculator"
+  calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
   input_stream: "ITERABLE:multi_hand_landmarks"
   output_stream: "ITEM:single_hand_landmarks"
   output_stream: "BATCH_END:landmark_timestamp"

mediapipe/docs/object_tracking_mobile_gpu.md

@@ -0,0 +1,475 @@
+# Object Detection and Tracking
+
+This doc focuses on the
+[below example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt)
+that performs object detection and tracking.
+Note that object detection is using TensorFlow Lite on GPU while tracking is using CPU.
+
+For overall context on object detection and tracking, please read this
+[Google Developer Blog](https://mediapipe.page.link/objecttrackingblog).
+
+![object_tracking_android_gpu_gif](images/mobile/object_tracking_android_gpu.gif)
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu)
+
+To build and install the app:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/objecttrackinggpu.apk
+```
+
+## Graph
+The object detection and tracking [main graph](#main-graph) internally utilizes a
+[object detection subgraph](#object-detection-subgraph), a
+[object tracking subgraph](#object-tracking-subgraph) and a
+[renderer subgraph](#renderer-subgraph).
+
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+
+![object_detection_mobile_gpu_graph](images/mobile/object_tracking_mobile_gpu.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt)
+
+```bash
+# MediaPipe graph that performs object detection and tracking.
+# Used in the examples in
+# mediapipie/examples/android/src/java/com/mediapipe/apps/objecttrackinggpu
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Resamples the images by specific frame rate. This calculator is used to
+# control the frequecy of subsequent calculators/subgraphs, e.g. less power
+# consumption for expensive process.
+node {
+  calculator: "PacketResamplerCalculator"
+  input_stream: "DATA:input_video"
+  output_stream: "DATA:throttled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.PacketResamplerCalculatorOptions] {
+      frame_rate: 0.5
+    }
+  }
+}
+
+# Subgraph that detections objects (see object_detection_gpu.pbtxt).
+node {
+  calculator: "ObjectDetectionSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "DETECTIONS:output_detections"
+}
+
+# Subgraph that tracks objects (see object_tracking.pbtxt).
+node {
+  calculator: "ObjectTrackingSubgraph"
+  input_stream: "VIDEO:input_video"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "DETECTIONS:tracked_detections"
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see renderer_gpu.pbtxt).
+node {
+  calculator: "RendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "DETECTIONS:tracked_detections"
+  output_stream: "IMAGE:output_video"
+}
+```
+
+### Object Detection Subgraph
+
+![object_detection_gpu_subgraph](images/mobile/object_detection_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_detection_gpu.pbtxt)
+
+```bash
+# MediaPipe object detection subgraph.
+
+type: "ObjectDetectionSubgraph"
+
+input_stream: "IMAGE:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+# Transforms the input image on GPU to a 320x320 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 320
+      output_height: 320
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS_GPU:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/ssdlite_object_detection.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  node_options: {
+    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 320
+      input_size_width: 320
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
+      num_classes: 91
+      num_boxes: 2034
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 3
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt"
+    }
+  }
+}
+```
+
+### Object Tracking Subgraph
+
+Object tracking subgraph uses a box tracking subgraph which is a generic
+tracking library that can be used for other use cases.
+
+![object_tracking_subgraph](images/mobile/object_tracking_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_tracking.pbtxt)
+
+```bash
+# MediaPipe object tracking subgraph.
+
+type: "ObjectTrackingSubgraph"
+
+input_stream: "VIDEO:input_video"
+input_stream: "DETECTIONS:new_detections"
+output_stream: "DETECTIONS:tracked_detections"
+
+# Assigns an unique id for each new detection.
+node {
+  calculator: "DetectionUniqueIdCalculator"
+  input_stream: "DETECTIONS:new_detections"
+  output_stream: "DETECTIONS:detections_with_id"
+}
+
+# Converts detections to TimedBox protos which are used as initial location
+# for tracking.
+node {
+  calculator: "DetectionsToTimedBoxListCalculator"
+  input_stream: "DETECTIONS:detections_with_id"
+  output_stream: "BOXES:start_pos"
+}
+
+# Subgraph that tracks boxes (see box_tracking.pbtxt).
+node {
+  calculator: "BoxTrackingSubgraph"
+  input_stream: "VIDEO:input_video"
+  input_stream: "BOXES:start_pos"
+  input_stream: "CANCEL_ID:cancel_object_id"
+  output_stream: "BOXES:boxes"
+}
+
+# Managers new detected objects and objects that are being tracked.
+# It associates the duplicated detections and updates the locations of
+# detections from tracking.
+node: {
+  calculator: "TrackedDetectionManagerCalculator"
+  input_stream: "DETECTIONS:detections_with_id"
+  input_stream: "TRACKING_BOXES:boxes"
+  output_stream: "DETECTIONS:tracked_detections"
+  output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "TRACKING_BOXES"
+        }
+        sync_set {
+          tag_index: "DETECTIONS"
+        }
+      }
+    }
+  }
+}
+
+```
+
+### Box Tracking Subgraph
+
+![box_tracking_subgraph](images/mobile/box_tracking_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking.pbtxt)
+
+```bash
+# MediaPipe box tracking subgraph.
+
+type: "BoxTrackingSubgraph"
+
+input_stream: "VIDEO:input_video"
+input_stream: "BOXES:start_pos"
+input_stream: "CANCEL_ID:cancel_object_id"
+output_stream: "BOXES:boxes"
+
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:downscaled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 240
+      output_height: 320
+    }
+  }
+}
+
+# Converts GPU buffer to ImageFrame for processing tracking.
+node: {
+  calculator: "GpuBufferToImageFrameCalculator"
+  input_stream: "downscaled_input_video"
+  output_stream: "downscaled_input_video_cpu"
+}
+
+# Performs motion analysis on an incoming video stream.
+node: {
+  calculator: "MotionAnalysisCalculator"
+  input_stream: "VIDEO:downscaled_input_video_cpu"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: {
+      analysis_options {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+        flow_options {
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options {
+            max_features: 500
+            adaptive_extraction_levels: 2
+            min_eig_val_settings {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+      }
+    }
+  }
+}
+
+# Reads optical flow fields defined in
+# mediapipe/framework/formats/motion/optical_flow_field.h,
+# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
+# to 0-255.
+node: {
+  calculator: "FlowPackagerCalculator"
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+# Tracks box positions over time.
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "TRACK_TIME:input_video"
+  input_stream: "START_POS:start_pos"
+  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  input_stream_info: {
+    tag_index: "CANCEL_OBJECT_ID"
+    back_edge: true
+  }
+  output_stream: "BOXES:boxes"
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "TRACKING"
+          tag_index: "TRACK_TIME"
+        }
+        sync_set {
+          tag_index: "START_POS"
+        }
+        sync_set {
+          tag_index: "CANCEL_OBJECT_ID"
+        }
+      }
+    }
+  }
+
+  node_options: {
+    [type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: {
+      tracker_options: {
+        track_step_options {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_SCALE
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}
+```
+
+### Renderer Subgraph
+
+![object_tracking_renderer_gpu_subgraph.pbtxt](images/mobile/object_tracking_renderer_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt)
+
+```bash
+# MediaPipe object tracking rendering subgraph.
+
+type: "RendererSubgraph"
+
+input_stream: "IMAGE:input_image"
+input_stream: "DETECTIONS:detections"
+output_stream: "IMAGE:output_image"
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:detections"
+  output_stream: "RENDER_DATA:detections_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+      render_detection_id: true
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "detections_render_data"
+  output_stream: "OUTPUT_FRAME_GPU:output_image"
+}
+
+```

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -75,6 +75,7 @@ android_library(
     resource_files = glob(["res/**"]),
     deps = [
         ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
         "//mediapipe/java/com/google/mediapipe/components:android_components",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java

@@ -17,18 +17,23 @@ package com.google.mediapipe.apps.handtrackinggpu;
 import android.graphics.SurfaceTexture;
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
 import com.google.mediapipe.components.CameraHelper;
 import com.google.mediapipe.components.CameraXPreviewHelper;
 import com.google.mediapipe.components.ExternalTextureConverter;
 import com.google.mediapipe.components.FrameProcessor;
 import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.glutil.EglManager;
+import com.google.protobuf.InvalidProtocolBufferException;
 
 /** Main activity of MediaPipe example apps. */
 public class MainActivity extends AppCompatActivity {
@@ -37,6 +42,8 @@ public class MainActivity extends AppCompatActivity {
   private static final String BINARY_GRAPH_NAME = "handtrackinggpu.binarypb";
   private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
   private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_HAND_PRESENCE_STREAM_NAME = "hand_presence";
+  private static final String OUTPUT_LANDMARKS_STREAM_NAME = "hand_landmarks";
   private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT;
 
   // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
@@ -90,6 +97,41 @@ public class MainActivity extends AppCompatActivity {
             OUTPUT_VIDEO_STREAM_NAME);
     processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
 
+    processor.addPacketCallback(
+        OUTPUT_HAND_PRESENCE_STREAM_NAME,
+        (packet) -> {
+          Boolean handPresence = PacketGetter.getBool(packet);
+          if (!handPresence) {
+            Log.d(
+                TAG,
+                "[TS:" + packet.getTimestamp() + "] Hand presence is false, no hands detected.");
+          }
+        });
+
+    processor.addPacketCallback(
+        OUTPUT_LANDMARKS_STREAM_NAME,
+        (packet) -> {
+          byte[] landmarksRaw = PacketGetter.getProtoBytes(packet);
+          try {
+            NormalizedLandmarkList landmarks = NormalizedLandmarkList.parseFrom(landmarksRaw);
+            if (landmarks == null) {
+              Log.d(TAG, "[TS:" + packet.getTimestamp() + "] No hand landmarks.");
+              return;
+            }
+            // Note: If hand_presence is false, these landmarks are useless.
+            Log.d(
+                TAG,
+                "[TS:"
+                    + packet.getTimestamp()
+                    + "] #Landmarks for hand: "
+                    + landmarks.getLandmarkCount());
+            Log.d(TAG, getLandmarksDebugString(landmarks));
+          } catch (InvalidProtocolBufferException e) {
+            Log.e(TAG, "Couldn't Exception received - " + e);
+            return;
+          }
+        });
+
     PermissionHelper.checkAndRequestCameraPermissions(this);
   }
 
@@ -164,4 +206,23 @@ public class MainActivity extends AppCompatActivity {
         });
     cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
   }
+
+  private static String getLandmarksDebugString(NormalizedLandmarkList landmarks) {
+    int landmarkIndex = 0;
+    String landmarksString = "";
+    for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
+      landmarksString +=
+          "\t\tLandmark["
+              + landmarkIndex
+              + "]: ("
+              + landmark.getX()
+              + ", "
+              + landmark.getY()
+              + ", "
+              + landmark.getZ()
+              + ")\n";
+      ++landmarkIndex;
+    }
+    return landmarksString;
+  }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -75,6 +75,7 @@ android_library(
     resource_files = glob(["res/**"]),
     deps = [
         ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
         "//mediapipe/java/com/google/mediapipe/components:android_components",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java

@@ -17,18 +17,23 @@ package com.google.mediapipe.apps.multihandtrackinggpu;
 import android.graphics.SurfaceTexture;
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
+import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
 import com.google.mediapipe.components.CameraHelper;
 import com.google.mediapipe.components.CameraXPreviewHelper;
 import com.google.mediapipe.components.ExternalTextureConverter;
 import com.google.mediapipe.components.FrameProcessor;
 import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.glutil.EglManager;
+import java.util.List;
 
 /** Main activity of MediaPipe example apps. */
 public class MainActivity extends AppCompatActivity {
@@ -37,6 +42,7 @@ public class MainActivity extends AppCompatActivity {
   private static final String BINARY_GRAPH_NAME = "multihandtrackinggpu.binarypb";
   private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
   private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_LANDMARKS_STREAM_NAME = "multi_hand_landmarks";
   private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT;
 
   // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
@@ -90,6 +96,20 @@ public class MainActivity extends AppCompatActivity {
             OUTPUT_VIDEO_STREAM_NAME);
     processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
 
+    processor.addPacketCallback(
+        OUTPUT_LANDMARKS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received multi-hand landmarks packet.");
+          List<NormalizedLandmarkList> multiHandLandmarks =
+              PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser());
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] "
+                  + getMultiHandLandmarksDebugString(multiHandLandmarks));
+        });
+
     PermissionHelper.checkAndRequestCameraPermissions(this);
   }
 
@@ -164,4 +184,32 @@ public class MainActivity extends AppCompatActivity {
         });
     cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
   }
+
+  private String getMultiHandLandmarksDebugString(List<NormalizedLandmarkList> multiHandLandmarks) {
+    if (multiHandLandmarks.isEmpty()) {
+      return "No hand landmarks";
+    }
+    String multiHandLandmarksStr = "Number of hands detected: " + multiHandLandmarks.size() + "\n";
+    int handIndex = 0;
+    for (NormalizedLandmarkList landmarks : multiHandLandmarks) {
+      multiHandLandmarksStr +=
+          "\t#Hand landmarks for hand[" + handIndex + "]: " + landmarks.getLandmarkCount() + "\n";
+      int landmarkIndex = 0;
+      for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
+        multiHandLandmarksStr +=
+            "\t\tLandmark ["
+                + landmarkIndex
+                + "]: ("
+                + landmark.getX()
+                + ", "
+                + landmark.getY()
+                + ", "
+                + landmark.getZ()
+                + ")\n";
+        ++landmarkIndex;
+      }
+      ++handIndex;
+    }
+    return multiHandLandmarksStr;
+  }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/AndroidManifest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.objecttrackinggpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD

@@ -0,0 +1,81 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/tracking:mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/tracking:mobile_gpu_binary_graph"],
+    outs = ["objecttrackinggpu.binarypb"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@androidx_concurrent_futures//jar",
+        "@androidx_lifecycle//jar",
+        "@com_google_guava_android//jar",
+    ],
+)
+
+android_binary(
+    name = "objecttrackinggpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.objecttrackinggpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/MainActivity.java

@@ -0,0 +1,167 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.objecttrackinggpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.glutil.EglManager;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "objecttrackinggpu.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java3");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/layout/activity_main.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Object Tracking GPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/styles.xml

@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>

mediapipe/examples/desktop/demo_run_graph_main.cc

@@ -76,6 +76,11 @@ DEFINE_string(output_video_path, "",
     RET_CHECK(writer.isOpened());
   } else {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
+#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
+    capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
+    capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
+    capture.set(cv::CAP_PROP_FPS, 30);
+#endif
   }
 
   LOG(INFO) << "Start running the calculator graph.";

mediapipe/examples/desktop/demo_run_graph_main_gpu.cc

@@ -86,6 +86,11 @@ DEFINE_string(output_video_path, "",
     RET_CHECK(writer.isOpened());
   } else {
     cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
+#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2)
+    capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
+    capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
+    capture.set(cv::CAP_PROP_FPS, 30);
+#endif
   }
 
   LOG(INFO) << "Start running the calculator graph.";

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -93,15 +93,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
 SPLITS = {
     "train": {
         "shards": 1000,
-        "examples": 541490
+        "examples": 541279
     },
     "validate": {
         "shards": 100,
-        "examples": 34715
+        "examples": 34688
     },
     "test": {
         "shards": 100,
-        "examples": 69321
+        "examples": 69278
     },
     "custom": {
         "csv": None,  # Add a CSV for your own data here.

mediapipe/examples/ios/handtrackinggpu/BUILD

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-licenses(["notice"])  # Apache 2.0
-
-MIN_IOS_VERSION = "10.0"
-
 load(
     "@build_bazel_rules_apple//apple:ios.bzl",
     "ios_application",
 )
 
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "10.0"
+
 # To use the 3D model instead of the default 2D model, add "--define 3D=true" to the
 # bazel build command.
 config_setting(
@@ -90,6 +90,7 @@ objc_library(
         "//mediapipe:ios_x86_64": [],
         "//conditions:default": [
             "//mediapipe/graphs/hand_tracking:mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
         ],
     }),
 )

mediapipe/examples/ios/handtrackinggpu/ViewController.mm

@@ -18,10 +18,13 @@
 #import "mediapipe/objc/MPPCameraInputSource.h"
 #import "mediapipe/objc/MPPLayerRenderer.h"
 
+#include "mediapipe/framework/formats/landmark.pb.h"
+
 static NSString* const kGraphName = @"hand_tracking_mobile_gpu";
 
 static const char* kInputStream = "input_video";
 static const char* kOutputStream = "output_video";
+static const char* kLandmarksOutputStream = "hand_landmarks";
 static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
 
 @interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
@@ -80,6 +83,7 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
   // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
   MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
   [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw];
   return newGraph;
 }
 
@@ -160,6 +164,25 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
   }
 }
 
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kLandmarksOutputStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No hand landmarks", packet.Timestamp().Value());
+      return;
+    }
+    const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>();
+    NSLog(@"[TS:%lld] Number of landmarks on hand: %d", packet.Timestamp().Value(),
+          landmarks.landmark_size());
+    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+      NSLog(@"\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+            landmarks.landmark(i).y(), landmarks.landmark(i).z());
+    }
+  }
+}
+
 #pragma mark - MPPInputSourceDelegate methods
 
 // Must be invoked on _videoQueue.

mediapipe/examples/ios/multihandtrackinggpu/BUILD

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-licenses(["notice"])  # Apache 2.0
-
-MIN_IOS_VERSION = "10.0"
-
 load(
     "@build_bazel_rules_apple//apple:ios.bzl",
     "ios_application",
 )
 
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "10.0"
+
 # To use the 3D model instead of the default 2D model, add "--define 3D=true" to the
 # bazel build command.
 config_setting(
@@ -90,6 +90,7 @@ objc_library(
         "//mediapipe:ios_x86_64": [],
         "//conditions:default": [
             "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
         ],
     }),
 )

mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm

@@ -18,10 +18,13 @@
 #import "mediapipe/objc/MPPCameraInputSource.h"
 #import "mediapipe/objc/MPPLayerRenderer.h"
 
+#include "mediapipe/framework/formats/landmark.pb.h"
+
 static NSString* const kGraphName = @"multi_hand_tracking_mobile_gpu";
 
 static const char* kInputStream = "input_video";
 static const char* kOutputStream = "output_video";
+static const char* kLandmarksOutputStream = "multi_hand_landmarks";
 static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
 
 @interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
@@ -80,6 +83,7 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
   // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
   MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
   [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw];
   return newGraph;
 }
 
@@ -160,6 +164,29 @@ static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
   }
 }
 
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kLandmarksOutputStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No hand landmarks", packet.Timestamp().Value());
+      return;
+    }
+    const auto& multi_hand_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+    NSLog(@"[TS:%lld] Number of hand instances with landmarks: %lu", packet.Timestamp().Value(),
+          multi_hand_landmarks.size());
+    for (int hand_index = 0; hand_index < multi_hand_landmarks.size(); ++hand_index) {
+      const auto& landmarks = multi_hand_landmarks[hand_index];
+      NSLog(@"\tNumber of landmarks for hand[%d]: %d", hand_index, landmarks.landmark_size());
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+              landmarks.landmark(i).y(), landmarks.landmark(i).z());
+      }
+    }
+  }
+}
+
 #pragma mark - MPPInputSourceDelegate methods
 
 // Must be invoked on _videoQueue.

mediapipe/framework/calculator_graph.cc

@@ -1094,6 +1094,19 @@ bool CalculatorGraph::IsNodeThrottled(int node_id) {
   return max_queue_size_ != -1 && !full_input_streams_[node_id].empty();
 }
 
+// Returns true if an input stream serves as a graph-output-stream.
+bool IsGraphOutputStream(
+    InputStreamManager* stream,
+    const std::vector<std::shared_ptr<internal::GraphOutputStream>>&
+        graph_output_streams) {
+  for (auto& graph_output_stream : graph_output_streams) {
+    if (stream == graph_output_stream->input_stream()) {
+      return true;
+    }
+  }
+  return false;
+}
+
 bool CalculatorGraph::UnthrottleSources() {
   // NOTE: We can be sure that this function will grow input streams enough
   // to unthrottle at least one source node.  The current stream queue sizes
@@ -1105,25 +1118,17 @@ bool CalculatorGraph::UnthrottleSources() {
   {
     absl::MutexLock lock(&full_input_streams_mutex_);
     for (absl::flat_hash_set<InputStreamManager*>& s : full_input_streams_) {
-      if (!s.empty()) {
-        full_streams.insert(s.begin(), s.end());
+      for (auto& stream : s) {
+        // The queue size of a graph output stream shouldn't change. Throttling
+        // should continue until the caller of the graph output stream consumes
+        // enough packets.
+        if (!IsGraphOutputStream(stream, graph_output_streams_)) {
+          full_streams.insert(stream);
+        }
       }
     }
   }
   for (InputStreamManager* stream : full_streams) {
-    // The queue size of a graph output stream shouldn't change. Throttling
-    // should continue until the caller of the graph output stream consumes
-    // enough packets.
-    bool is_graph_output_stream = false;
-    for (auto& graph_output_stream : graph_output_streams_) {
-      if (stream == graph_output_stream->input_stream()) {
-        is_graph_output_stream = true;
-        break;
-      }
-    }
-    if (is_graph_output_stream) {
-      continue;
-    }
     if (Config().report_deadlock()) {
       RecordError(::mediapipe::UnavailableError(absl::StrCat(
           "Detected a deadlock due to input throttling for: \"", stream->Name(),

mediapipe/framework/calculator_graph_event_loop_test.cc

@@ -541,5 +541,90 @@ TEST_F(CalculatorGraphEventLoopTest, WaitToAddPacketToInputStream) {
   ASSERT_EQ(kNumInputPackets, output_packets_.size());
 }
 
+// Captures log messages during testing.
+class TextMessageLogSink : public LogSink {
+ public:
+  std::vector<std::string> messages;
+  void Send(const LogEntry& entry) {
+    messages.push_back(std::string(entry.text_message()));
+  }
+};
+
+// Verifies that CalculatorGraph::UnthrottleSources does not run repeatedly
+// in a "busy-loop" while the graph is throttled due to a graph-output stream.
+TEST_F(CalculatorGraphEventLoopTest, UnthrottleSources) {
+  CalculatorGraphConfig graph_config;
+  ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(
+      R"(
+          node {
+            calculator: "PassThroughCalculator"
+            input_stream: "input_numbers"
+            output_stream: "output_numbers"
+          }
+          input_stream: "input_numbers"
+          output_stream: "output_numbers"
+          num_threads: 2
+          max_queue_size: 5
+      )",
+      &graph_config));
+  constexpr int kQueueSize = 5;
+
+  // Initialize and start the mediapipe graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  graph.SetGraphInputStreamAddMode(
+      CalculatorGraph::GraphInputStreamAddMode::ADD_IF_NOT_FULL);
+  auto poller_status = graph.AddOutputStreamPoller("output_numbers");
+  MP_ASSERT_OK(poller_status.status());
+  mediapipe::OutputStreamPoller& poller = poller_status.ValueOrDie();
+  poller.SetMaxQueueSize(kQueueSize);
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Lambda that adds a packet to the calculator graph.
+  auto add_packet = [&graph](std::string s, int i) {
+    return graph.AddPacketToInputStream(s, MakePacket<int>(i).At(Timestamp(i)));
+  };
+
+  // Start capturing VLOG messages from the mediapipe::Scheduler.
+  TextMessageLogSink log_listener;
+  mediapipe::AddLogSink(&log_listener);
+  SetVLOGLevel("scheduler", 3);
+
+  // Add just enough packets to fill the output stream queue.
+  std::vector<Packet> out_packets;
+  for (int i = 0; i < kQueueSize; ++i) {
+    MP_EXPECT_OK(add_packet("input_numbers", i));
+    MP_EXPECT_OK(graph.WaitUntilIdle());
+  }
+
+  // The graph is throttled due to the full output stream.
+  EXPECT_FALSE(add_packet("input_numbers", kQueueSize).ok());
+
+  // CalculatorGraph::UnthrottleSources should be called just one time.
+  absl::SleepFor(absl::Milliseconds(100));
+
+  // Read all packets from the output stream queue and close the graph.
+  for (int i = 0; i < kQueueSize; ++i) {
+    Packet packet;
+    EXPECT_TRUE(poller.Next(&packet));
+    out_packets.push_back(packet);
+  }
+  MP_EXPECT_OK(graph.CloseAllInputStreams());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+  EXPECT_EQ(kQueueSize, out_packets.size());
+
+  // Stop capturing VLOG messages.
+  SetVLOGLevel("scheduler", 0);
+  mediapipe::RemoveLogSink(&log_listener);
+
+  // Count and validate the calls to UnthrottleSources.
+  int loop_count = 0;
+  for (auto& message : log_listener.messages) {
+    loop_count += (message == "HandleIdle: unthrottling") ? 1 : 0;
+  }
+  EXPECT_GE(loop_count, 1);
+  EXPECT_LE(loop_count, 2);
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/formats/BUILD

@@ -13,10 +13,6 @@
 # limitations under the License.
 #
 
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 
 package(
@@ -24,6 +20,10 @@ package(
     features = ["-layering_check"],
 )
 
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
 proto_library(
     name = "detection_proto",
     srcs = ["detection.proto"],
@@ -241,7 +241,9 @@ cc_test(
 proto_library(
     name = "rect_proto",
     srcs = ["rect.proto"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//visibility:public",
+    ],
 )
 
 mediapipe_cc_proto_library(
@@ -253,10 +255,21 @@ mediapipe_cc_proto_library(
     deps = [":rect_proto"],
 )
 
+java_lite_proto_library(
+    name = "rect_java_proto_lite",
+    strict_deps = 0,
+    visibility = [
+        "//mediapipe:__subpackages__",
+    ],
+    deps = [":landmark_proto"],
+)
+
 proto_library(
     name = "landmark_proto",
     srcs = ["landmark.proto"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//visibility:public",
+    ],
 )
 
 mediapipe_cc_proto_library(
@@ -265,3 +278,19 @@ mediapipe_cc_proto_library(
     visibility = ["//visibility:public"],
     deps = [":landmark_proto"],
 )
+
+java_lite_proto_library(
+    name = "landmark_java_proto_lite",
+    strict_deps = 0,
+    visibility = [
+        "//mediapipe:__subpackages__",
+    ],
+    deps = [":landmark_proto"],
+)
+
+# Expose the proto source files for building mediapipe AAR.
+filegroup(
+    name = "protos_src",
+    srcs = glob(["*.proto"]),
+    visibility = ["//mediapipe:__subpackages__"],
+)

mediapipe/framework/formats/annotation/rasterization.proto

@@ -27,7 +27,7 @@ message Rasterization {
     required int32 right_x = 3;
   }
 
-  // Intervals are always sorted by y-corrdinate.
+  // Intervals are always sorted by y-coordinate.
   // Therefore, a region occupies a set of scanlines ranging
   // from interval(0).y() to interval(interval_size() - 1)).y().
   // Note: In video, at some scanlines no interval might be present.

mediapipe/framework/formats/landmark.proto

@@ -16,6 +16,9 @@ syntax = "proto2";
 
 package mediapipe;
 
+option java_package = "com.google.mediapipe.formats.proto";
+option java_outer_classname = "LandmarkProto";
+
 // A landmark that can have 1 to 3 dimensions. Use x for 1D points, (x, y) for
 // 2D points and (x, y, z) for 3D points. For more dimensions, consider using
 // matrix_data.proto.

mediapipe/framework/packet.cc

@@ -107,6 +107,14 @@ const proto_ns::MessageLite& Packet::GetProtoMessageLite() const {
   return *proto;
 }
 
+StatusOr<std::vector<const proto_ns::MessageLite*>>
+Packet::GetVectorOfProtoMessageLitePtrs() {
+  if (holder_ == nullptr) {
+    return ::mediapipe::InternalError("Packet is empty.");
+  }
+  return holder_->GetVectorOfProtoMessageLite();
+}
+
 MEDIAPIPE_REGISTER_TYPE(::mediapipe::Packet, "::mediapipe::Packet", nullptr,
                         nullptr);
 MEDIAPIPE_REGISTER_TYPE(::std::vector<::mediapipe::Packet>,

mediapipe/framework/packet.h

@@ -163,6 +163,13 @@ class Packet {
   // object type is protocol buffer, crashes otherwise.
   const proto_ns::MessageLite& GetProtoMessageLite() const;
 
+  // Returns a vector of pointers to MessageLite data, if the underlying
+  // object type is a vector of MessageLite data, returns an error otherwise.
+  // Note: This function is meant to be used internally within the MediaPipe
+  // framework only.
+  StatusOr<std::vector<const proto_ns::MessageLite*>>
+  GetVectorOfProtoMessageLitePtrs();
+
   // Returns an error if the packet does not contain data of type T.
   template <typename T>
   ::mediapipe::Status ValidateAsType() const;
@@ -347,6 +354,12 @@ class HolderBase {
   // underlying object is protocol buffer type, otherwise, nullptr is returned.
   virtual const proto_ns::MessageLite* GetProtoMessageLite() = 0;
 
+  // Returns a vector<MessageLite*> for the data in the holder, if the
+  // underlying object is a vector of protocol buffer objects, otherwise,
+  // returns an error.
+  virtual StatusOr<std::vector<const proto_ns::MessageLite*>>
+  GetVectorOfProtoMessageLite() = 0;
+
  private:
   size_t type_id_;
 };
@@ -364,6 +377,37 @@ const proto_ns::MessageLite* ConvertToProtoMessageLite(const T* data,
   return data;
 }
 
+// Helper structs for determining if a type is an std::vector<Proto>.
+template <typename Type>
+struct is_proto_vector : public std::false_type {};
+
+template <typename ItemT, typename Allocator>
+struct is_proto_vector<std::vector<ItemT, Allocator>>
+    : public std::is_base_of<proto_ns::MessageLite, ItemT>::type {};
+
+// Helper function to create and return a vector of pointers to proto message
+// elements of the vector passed into the function.
+template <typename T>
+StatusOr<std::vector<const proto_ns::MessageLite*>>
+ConvertToVectorOfProtoMessageLitePtrs(const T* data,
+                                      /*is_proto_vector=*/std::false_type) {
+  return ::mediapipe::InvalidArgumentError(absl::StrCat(
+      "The Packet stores \"", typeid(T).name(), "\"",
+      "which is not convertible to vector<proto_ns::MessageLite*>."));
+}
+
+template <typename T>
+StatusOr<std::vector<const proto_ns::MessageLite*>>
+ConvertToVectorOfProtoMessageLitePtrs(const T* data,
+                                      /*is_proto_vector=*/std::true_type) {
+  std::vector<const proto_ns::MessageLite*> result;
+  for (auto it = data->begin(); it != data->end(); ++it) {
+    const proto_ns::MessageLite* element = &(*it);
+    result.push_back(element);
+  }
+  return result;
+}
+
 template <typename T>
 class Holder : public HolderBase {
  public:
@@ -421,6 +465,14 @@ class Holder : public HolderBase {
         ptr_, std::is_base_of<proto_ns::MessageLite, T>());
   }
 
+  // Returns a vector<MessageLite*> for the data in the holder, if the
+  // underlying object is a vector of protocol buffer objects, otherwise,
+  // returns an error.
+  StatusOr<std::vector<const proto_ns::MessageLite*>>
+  GetVectorOfProtoMessageLite() override {
+    return ConvertToVectorOfProtoMessageLitePtrs(ptr_, is_proto_vector<T>());
+  }
+
  private:
   // Call delete[] if T is an array, delete otherwise.
   template <typename U = T>

mediapipe/framework/port.h

@@ -37,12 +37,16 @@
 #if !defined(MEDIAPIPE_IOS) && !TARGET_OS_OSX
 #define MEDIAPIPE_IOS
 #endif
+#if !defined(MEDIAPIPE_OSX) && TARGET_OS_OSX
+#define MEDIAPIPE_OSX
+#endif
 #endif
 
 // These platforms do not support OpenGL ES Compute Shaders (v3.1 and up),
-// but can still run OpenGL ES 3.0 and below.
-#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
-    (defined(__APPLE__) || defined(__EMSCRIPTEN__))
+// but may or may not still be able to run other OpenGL code.
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) &&         \
+    (defined(__APPLE__) || defined(__EMSCRIPTEN__) || \
+     defined(MEDIAPIPE_DISABLE_GPU))
 #define MEDIAPIPE_DISABLE_GL_COMPUTE
 #endif
 

mediapipe/framework/port/BUILD

@@ -176,6 +176,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:port",
         "@com_github_glog_glog//:glog",
+        "@com_google_absl//absl/time",
     ],
 )
 

mediapipe/framework/port/logging.h

@@ -15,6 +15,50 @@
 #ifndef MEDIAPIPE_PORT_LOGGING_H_
 #define MEDIAPIPE_PORT_LOGGING_H_
 
+#include "absl/time/time.h"
 #include "glog/logging.h"
 
+namespace mediapipe {
+using LogSeverity = google::LogSeverity;
+const auto SetVLOGLevel = google::SetVLOGLevel;
+class LogEntry {
+ public:
+  LogEntry(LogSeverity severity, const struct ::tm* tm_time,
+           absl::string_view message)
+      : severity_(severity),
+        timestamp_(absl::FromTM(*tm_time, absl::LocalTimeZone())),
+        text_message_(message) {}
+  LogSeverity log_severity() const { return severity_; }
+  absl::Time timestamp() const { return timestamp_; }
+  absl::string_view text_message() const { return text_message_; }
+
+ private:
+  LogSeverity severity_;
+  absl::Time timestamp_;
+  absl::string_view text_message_;
+};
+class LogSink : public google::LogSink {
+ public:
+  virtual ~LogSink() = default;
+  virtual void Send(const LogEntry& entry) = 0;
+  virtual void WaitTillSent() {}
+
+ private:
+  virtual void send(LogSeverity severity, const char* full_filename,
+                    const char* base_filename, int line,
+                    const struct ::tm* tm_time, const char* message,
+                    size_t message_len) {
+    LogEntry log_entry(severity, tm_time,
+                       absl::string_view(message, message_len));
+    Send(log_entry);
+  }
+};
+inline void AddLogSink(LogSink* destination) {
+  google::AddLogSink(destination);
+}
+inline void RemoveLogSink(LogSink* destination) {
+  google::RemoveLogSink(destination);
+}
+}  // namespace mediapipe
+
 #endif  // MEDIAPIPE_PORT_LOGGING_H_

mediapipe/framework/port/opencv_core_inc.h

@@ -20,6 +20,8 @@
 #ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
 #include <opencv2/core/core.hpp>
 #else
+#include <opencv2/cvconfig.h>
+
 #include <opencv2/core.hpp>
 #endif