Project import generated by Copybara.

GitOrigin-RevId: 0517756260533d374df93679965ca662d0ec6943

.bazelrc

@@ -12,13 +12,16 @@ build --copt='-Wno-comment'
 build --copt='-Wno-return-type'
 build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
-# Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
+# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
 build --incompatible_disable_deprecated_attr_params=false
 build --incompatible_depset_is_not_iterable=false
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
 
+# Allow debugging with XCODE
+build --apple_generate_dsym
+
 # Android configs.
 build:android --crosstool_top=//external:android/crosstool
 build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain

Dockerfile

@@ -35,6 +35,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libopencv-highgui-dev \
         libopencv-imgproc-dev \
         libopencv-video-dev \
+        libopencv-calib3d-dev \
+        libopencv-features2d-dev \
         software-properties-common && \
     add-apt-repository -y ppa:openjdk-r/ppa && \
     apt-get update && apt-get install -y openjdk-8-jdk && \

README.md

@@ -14,12 +14,14 @@
 * [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
 * [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
+* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
+* [AutoFlip](mediapipe/docs/autoflip.md)
 
 ![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
 ![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
-![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
+![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
 
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).

WORKSPACE

@@ -10,7 +10,9 @@ http_archive(
     sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.24.1")
+versions.check(minimum_bazel_version = "0.24.1",
+               maximum_bazel_version = "1.2.1")
+
 
 # ABSL cpp library lts_2019_08_08.
 http_archive(
@@ -116,6 +118,13 @@ http_archive(
       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
+    # Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
+    patches = [
+        "@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
     sha256 = _TENSORFLOW_SHA256,
 )
@@ -123,8 +132,22 @@ http_archive(
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
 tf_workspace(tf_repo_name = "org_tensorflow")
 
+http_archive(
+    name = "ceres_solver",
+    url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
+    patches = [
+        "@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+    strip_prefix = "ceres-solver-1.14.0",
+    sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
+)
+
 # Please run
 # $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+#                        libopencv-calib3d-dev libopencv-features2d-dev \
 #                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
@@ -292,3 +315,4 @@ http_archive(
     strip_prefix = "google-toolbox-for-mac-2.2.1",
     build_file = "@//third_party:google_toolbox_for_mac.BUILD",
 )
+

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -9,6 +9,7 @@
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
     "mediapipe/examples/ios/handdetectiongpu/BUILD",
     "mediapipe/examples/ios/handtrackinggpu/BUILD",
+    "mediapipe/examples/ios/multihandtrackinggpu/BUILD",
     "mediapipe/examples/ios/objectdetectioncpu/BUILD",
     "mediapipe/examples/ios/objectdetectiongpu/BUILD"
   ],
@@ -18,6 +19,7 @@
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
     "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
     "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
+    "//mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp",
     "//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
     "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
     "//mediapipe/objc:mediapipe_framework_ios"
@@ -84,6 +86,8 @@
     "mediapipe/examples/ios/handdetectiongpu/Base.lproj",
     "mediapipe/examples/ios/handtrackinggpu",
     "mediapipe/examples/ios/handtrackinggpu/Base.lproj",
+    "mediapipe/examples/ios/multihandtrackinggpu",
+    "mediapipe/examples/ios/multihandtrackinggpu/Base.lproj",
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectioncpu/Base.lproj",
     "mediapipe/examples/ios/objectdetectiongpu",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -16,6 +16,7 @@
     "mediapipe/examples/ios/facedetectiongpu",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",
+    "mediapipe/examples/ios/multihandtrackinggpu",
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectiongpu"
   ],

mediapipe/calculators/core/BUILD

@@ -47,6 +47,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "packet_thinner_calculator_proto",
+    srcs = ["packet_thinner_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "split_vector_calculator_proto",
     srcs = ["split_vector_calculator.proto"],
@@ -102,6 +109,14 @@ mediapipe_cc_proto_library(
     deps = [":packet_resampler_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "packet_thinner_calculator_cc_proto",
+    srcs = ["packet_thinner_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":packet_thinner_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "split_vector_calculator_cc_proto",
     srcs = ["split_vector_calculator.proto"],
@@ -165,6 +180,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
 )
@@ -283,7 +299,6 @@ cc_test(
     srcs = ["concatenate_vector_calculator_test.cc"],
     deps = [
         ":concatenate_vector_calculator",
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:timestamp",
@@ -450,6 +465,37 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "packet_thinner_calculator",
+    srcs = ["packet_thinner_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "packet_thinner_calculator_test",
+    srcs = ["packet_thinner_calculator_test.cc"],
+    deps = [
+        ":packet_thinner_calculator",
+        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "pass_through_calculator",
     srcs = ["pass_through_calculator.cc"],
@@ -571,6 +617,7 @@ cc_test(
 cc_library(
     name = "packet_resampler_calculator",
     srcs = ["packet_resampler_calculator.cc"],
+    hdrs = ["packet_resampler_calculator.h"],
     visibility = [
         "//visibility:public",
     ],
@@ -594,17 +641,17 @@ cc_library(
 cc_test(
     name = "packet_resampler_calculator_test",
     timeout = "short",
-    srcs = ["packet_resampler_calculator_test.cc"],
+    srcs = [
+        "packet_resampler_calculator_test.cc",
+    ],
     deps = [
         ":packet_resampler_calculator",
         "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:video_stream_header",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
-        "//mediapipe/framework/port:status",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -697,7 +744,13 @@ cc_library(
         "//mediapipe/util:resource_util",
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
-    ],
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+        ],
+    }),
     alwayslink = 1,
 )
 

mediapipe/calculators/core/add_header_calculator.cc

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/logging.h"
 
 namespace mediapipe {
 
-// Attach the header from one stream to another stream.
+// Attach the header from a stream or side input to another stream.
 //
 // The header stream (tag HEADER) must not have any packets in it.
 //
@@ -25,17 +26,53 @@ namespace mediapipe {
 // calculator to not need a header or to accept a separate stream with
 // a header, that would be more future proof.
 //
+// Example usage 1:
+// node {
+//   calculator: "AddHeaderCalculator"
+//   input_stream: "DATA:audio"
+//   input_stream: "HEADER:audio_header"
+//   output_stream: "audio_with_header"
+// }
+//
+// Example usage 2:
+// node {
+//   calculator: "AddHeaderCalculator"
+//   input_stream: "DATA:audio"
+//   input_side_packet: "HEADER:audio_header"
+//   output_stream: "audio_with_header"
+// }
+//
 class AddHeaderCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("HEADER").SetNone();
+    bool has_side_input = false;
+    bool has_header_stream = false;
+    if (cc->InputSidePackets().HasTag("HEADER")) {
+      cc->InputSidePackets().Tag("HEADER").SetAny();
+      has_side_input = true;
+    }
+    if (cc->Inputs().HasTag("HEADER")) {
+      cc->Inputs().Tag("HEADER").SetNone();
+      has_header_stream = true;
+    }
+    if (has_side_input == has_header_stream) {
+      return mediapipe::InvalidArgumentError(
+          "Header must be provided via exactly one of side input and input "
+          "stream");
+    }
     cc->Inputs().Tag("DATA").SetAny();
     cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    const Packet& header = cc->Inputs().Tag("HEADER").Header();
+    Packet header;
+    if (cc->InputSidePackets().HasTag("HEADER")) {
+      header = cc->InputSidePackets().Tag("HEADER");
+    }
+    if (cc->Inputs().HasTag("HEADER")) {
+      header = cc->Inputs().Tag("HEADER").Header();
+    }
     if (!header.IsEmpty()) {
       cc->Outputs().Index(0).SetHeader(header);
     }

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -14,8 +14,10 @@
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/timestamp.h"
 #include "mediapipe/framework/tool/validate_type.h"
@@ -24,7 +26,7 @@ namespace mediapipe {
 
 class AddHeaderCalculatorTest : public ::testing::Test {};
 
-TEST_F(AddHeaderCalculatorTest, Works) {
+TEST_F(AddHeaderCalculatorTest, HeaderStream) {
   CalculatorGraphConfig::Node node;
   node.set_calculator("AddHeaderCalculator");
   node.add_input_stream("HEADER:header_stream");
@@ -96,4 +98,62 @@ TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
   ASSERT_FALSE(runner.Run().ok());
 }
 
+TEST_F(AddHeaderCalculatorTest, InputSidePacket) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+  node.add_input_side_packet("HEADER:header");
+
+  CalculatorRunner runner(node);
+
+  // Set header and add 5 packets.
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run calculator.
+  MP_ASSERT_OK(runner.Run());
+
+  ASSERT_EQ(1, runner.Outputs().NumEntries());
+
+  // Test output.
+  EXPECT_EQ(std::string("my_header"),
+            runner.Outputs().Index(0).header.Get<std::string>());
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(5, output_packets.size());
+  for (int i = 0; i < 5; ++i) {
+    const int val = output_packets[i].Get<int>();
+    EXPECT_EQ(i, val);
+    EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp());
+  }
+}
+
+TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("HEADER:header_stream");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+  node.add_input_side_packet("HEADER:header");
+
+  CalculatorRunner runner(node);
+
+  // Set both headers and add 5 packets.
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  runner.MutableSidePackets()->Tag("HEADER") =
+      Adopt(new std::string("my_header"));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run should fail because header can only be provided one way.
+  EXPECT_EQ(runner.Run().code(), ::mediapipe::InvalidArgumentError("").code());
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -12,23 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cstdlib>
+#include "mediapipe/calculators/core/packet_resampler_calculator.h"
-#include <memory>
-#include <string>
 
-#include "absl/strings/str_cat.h"
+#include <memory>
-#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/deps/mathutil.h"
-#include "mediapipe/framework/deps/random_base.h"
-#include "mediapipe/framework/formats/video_stream_header.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/status_macros.h"
-#include "mediapipe/framework/tool/options_util.h"
 
 namespace {
 
@@ -45,120 +31,7 @@ std::unique_ptr<RandomBase> CreateSecureRandom(const std::string& seed) {
 
 namespace mediapipe {
 
-// This calculator is used to normalize the frequency of the packets
-// out of a stream. Given a desired frame rate, packets are going to be
-// removed or added to achieve it.
-//
-// The jitter feature is disabled by default. To enable it, you need to
-// implement CreateSecureRandom(const std::string&).
-//
-// The data stream may be either specified as the only stream (by index)
-// or as the stream with tag "DATA".
-//
-// The input and output streams may be accompanied by a VIDEO_HEADER
-// stream.  This stream includes a VideoHeader at Timestamp::PreStream().
-// The input VideoHeader on the VIDEO_HEADER stream will always be updated
-// with the resampler frame rate no matter what the options value for
-// output_header is before being output on the output VIDEO_HEADER stream.
-// If the input VideoHeader is not available, then only the frame rate
-// value will be set in the output.
-//
-// Related:
-//   packet_downsampler_calculator.cc: skips packets regardless of timestamps.
-class PacketResamplerCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  ::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
-
-  // Given the current count of periods that have passed, this returns
-  // the next valid timestamp of the middle point of the next period:
-  //    if count is 0, it returns the first_timestamp_.
-  //    if count is 1, it returns the first_timestamp_ + period (corresponding
-  //       to the first tick using exact fps)
-  // e.g. for frame_rate=30 and first_timestamp_=0:
-  //    0: 0
-  //    1: 33333
-  //    2: 66667
-  //    3: 100000
-  //
-  // Can only be used if jitter_ equals zero.
-  Timestamp PeriodIndexToTimestamp(int64 index) const;
-
-  // Given a Timestamp, finds the closest sync Timestamp based on
-  // first_timestamp_ and the desired fps.
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 TimestampToPeriodIndex(Timestamp timestamp) const;
-
-  // Outputs a packet if it is in range (start_time_, end_time_).
-  void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
-
-  // The timestamp of the first packet received.
-  Timestamp first_timestamp_;
-
-  // Number of frames per second (desired output frequency).
-  double frame_rate_;
-
-  // Inverse of frame_rate_.
-  int64 frame_time_usec_;
-
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
-  VideoHeader video_header_;
-  // The "DATA" input stream.
-  CollectionItemId input_data_id_;
-  // The "DATA" output stream.
-  CollectionItemId output_data_id_;
-
-  // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
-  bool flush_last_packet_;
-
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
-  double jitter_ = 0.0;
-  Timestamp next_output_timestamp_;
-
-  // If specified, output timestamps are aligned with base_timestamp.
-  // Otherwise, they are aligned with the first input timestamp.
-  Timestamp base_timestamp_;
-
-  // If specified, only outputs at/after start_time are included.
-  Timestamp start_time_;
-
-  // If specified, only outputs before end_time are included.
-  Timestamp end_time_;
-
-  // If set, the output timestamps nearest to start_time and end_time
-  // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.
-  bool round_limits_;
-};
-
 REGISTER_CALCULATOR(PacketResamplerCalculator);
-
 namespace {
 // Returns a TimestampDiff (assuming microseconds) corresponding to the
 // given time in seconds.
@@ -279,7 +152,10 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
           "SecureRandom is not available.  With \"jitter\" specified, "
           "PacketResamplerCalculator processing cannot proceed.");
     }
+    packet_reservoir_random_ = CreateSecureRandom(seed);
   }
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
   return ::mediapipe::OkStatus();
 }
 
@@ -294,6 +170,14 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
     }
   }
   if (jitter_ != 0.0 && random_ != nullptr) {
+    // Packet reservior is used to make sure there's an output for every period,
+    // e.g. partial period at the end of the stream.
+    if (packet_reservoir_->IsEnabled() &&
+        (first_timestamp_ == Timestamp::Unset() ||
+         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
+      packet_reservoir_->AddSample(curr_packet);
+    }
     MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
   } else {
     MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
@@ -303,11 +187,14 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 }
 
 void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
   next_output_timestamp_ =
       first_timestamp_ + frame_time_usec_ * random_->RandFloat();
 }
 
 void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  packet_reservoir_->Disable();
   next_output_timestamp_ +=
       frame_time_usec_ *
       ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
@@ -330,22 +217,27 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
     return ::mediapipe::OkStatus();
   }
 
-  LOG_IF(WARNING, frame_time_usec_ <
+  if (frame_time_usec_ <
-                      (cc->InputTimestamp() - last_packet_.Timestamp()).Value())
+      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
-      << "Adding jitter is meaningless when upsampling.";
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
 
-  const int64 curr_diff =
+  while (true) {
-      (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    const int64 last_diff =
-  const int64 last_diff =
+        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-      (next_output_timestamp_ - last_packet_.Timestamp()).Value();
+    RET_CHECK_GT(last_diff, 0);
-  if (curr_diff * last_diff > 0) {
+    const int64 curr_diff =
-    return ::mediapipe::OkStatus();
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
+                                ? last_packet_
+                                : cc->Inputs().Get(input_data_id_).Value())
+                               .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
   }
-  OutputWithinLimits(cc, (std::abs(curr_diff) > std::abs(last_diff)
-                              ? last_packet_
-                              : cc->Inputs().Get(input_data_id_).Value())
-                             .At(next_output_timestamp_));
-  UpdateNextOutputTimestampWithJitter();
   return ::mediapipe::OkStatus();
 }
 
@@ -426,6 +318,9 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
     OutputWithinLimits(cc,
                        last_packet_.At(PeriodIndexToTimestamp(period_count_)));
   }
+  if (!packet_reservoir_->IsEmpty()) {
+    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -0,0 +1,168 @@
+#ifndef MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/deps/mathutil.h"
+#include "mediapipe/framework/deps/random_base.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/tool/options_util.h"
+
+namespace mediapipe {
+
+class PacketReservoir {
+ public:
+  PacketReservoir(RandomBase* rng) : rng_(rng) {}
+  // Replace candidate with current packet with 1/count_ probability.
+  void AddSample(Packet sample) {
+    if (rng_->UnbiasedUniform(++count_) == 0) {
+      reservoir_ = sample;
+    }
+  }
+  bool IsEnabled() { return rng_ && enabled_; }
+  void Disable() {
+    if (enabled_) enabled_ = false;
+  }
+  void Clear() { count_ = 0; }
+  bool IsEmpty() { return count_ == 0; }
+  Packet GetSample() { return reservoir_; }
+
+ private:
+  RandomBase* rng_;
+  bool enabled_ = true;
+  int32 count_ = 0;
+  Packet reservoir_;
+};
+
+// This calculator is used to normalize the frequency of the packets
+// out of a stream. Given a desired frame rate, packets are going to be
+// removed or added to achieve it.
+//
+// The jitter feature is disabled by default. To enable it, you need to
+// implement CreateSecureRandom(const std::string&).
+//
+// The data stream may be either specified as the only stream (by index)
+// or as the stream with tag "DATA".
+//
+// The input and output streams may be accompanied by a VIDEO_HEADER
+// stream.  This stream includes a VideoHeader at Timestamp::PreStream().
+// The input VideoHeader on the VIDEO_HEADER stream will always be updated
+// with the resampler frame rate no matter what the options value for
+// output_header is before being output on the output VIDEO_HEADER stream.
+// If the input VideoHeader is not available, then only the frame rate
+// value will be set in the output.
+//
+// Related:
+//   packet_downsampler_calculator.cc: skips packets regardless of timestamps.
+class PacketResamplerCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestampWithJitter();
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Logic for Process() when jitter_ != 0.0.
+  ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
+
+  // Logic for Process() when jitter_ == 0.0.
+  ::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
+
+  // Given the current count of periods that have passed, this returns
+  // the next valid timestamp of the middle point of the next period:
+  //    if count is 0, it returns the first_timestamp_.
+  //    if count is 1, it returns the first_timestamp_ + period (corresponding
+  //       to the first tick using exact fps)
+  // e.g. for frame_rate=30 and first_timestamp_=0:
+  //    0: 0
+  //    1: 33333
+  //    2: 66667
+  //    3: 100000
+  //
+  // Can only be used if jitter_ equals zero.
+  Timestamp PeriodIndexToTimestamp(int64 index) const;
+
+  // Given a Timestamp, finds the closest sync Timestamp based on
+  // first_timestamp_ and the desired fps.
+  //
+  // Can only be used if jitter_ equals zero.
+  int64 TimestampToPeriodIndex(Timestamp timestamp) const;
+
+  // Outputs a packet if it is in range (start_time_, end_time_).
+  void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Number of frames per second (desired output frequency).
+  double frame_rate_;
+
+  // Inverse of frame_rate_.
+  int64 frame_time_usec_;
+
+  // Number of periods that have passed (= #packets sent to the output).
+  //
+  // Can only be used if jitter_ equals zero.
+  int64 period_count_;
+
+  // The last packet that was received.
+  Packet last_packet_;
+
+  VideoHeader video_header_;
+  // The "DATA" input stream.
+  CollectionItemId input_data_id_;
+  // The "DATA" output stream.
+  CollectionItemId output_data_id_;
+
+  // Indicator whether to flush last packet even if its timestamp is greater
+  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  bool flush_last_packet_;
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+  double jitter_ = 0.0;
+  Timestamp next_output_timestamp_;
+  Timestamp next_output_timestamp_min_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+
+  // If specified, only outputs at/after start_time are included.
+  Timestamp start_time_;
+
+  // If specified, only outputs before end_time are included.
+  Timestamp end_time_;
+
+  // If set, the output timestamps nearest to start_time and end_time
+  // are included in the output, even if the nearest timestamp is not
+  // between start_time and end_time.W
+  bool round_limits_;
+
+  // packet reservior used for sampling random packet out of partial
+  // period when jitter is enabled
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/core/packet_resampler_calculator.h"
+
 #include <memory>
 #include <string>
 #include <vector>
@@ -29,7 +31,6 @@
 namespace mediapipe {
 
 namespace {
-
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
 // against expected outputs (both timestamps and contents).

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -0,0 +1,304 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Declaration of PacketThinnerCalculator.
+
+#include <cmath>  // for ceil
+#include <memory>
+
+#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+const double kTimebaseUs = 1000000;  // Microseconds.
+const char* const kPeriodTag = "PERIOD";
+}  // namespace
+
+// This calculator is used to thin an input stream of Packets.
+// An example application would be to sample decoded frames of video
+// at a coarser temporal resolution. Unless otherwise stated, all
+// timestamps are in units of microseconds.
+//
+// Thinning can be accomplished in one of two ways:
+// 1) asynchronous thinning (known below as async):
+//    Algorithm does not rely on a master clock and is parameterized only
+//    by a single option -- the period.  Once a packet is emitted, the
+//    thinner will discard subsequent packets for the duration of the period
+//    [Analogous to a refractory period during which packet emission is
+//    suppressed.]
+//    Packets arriving before start_time are discarded, as are packets
+//    arriving at or after end_time.
+// 2) synchronous thinning (known below as sync):
+//    There are two variants of this algorithm, both parameterized by a
+//    start_time and a period.  As in (1), packets arriving before start_time
+//    or at/after end_time are discarded.  Otherwise, at most one packet is
+//    emitted during a period, centered at timestamps generated by the
+//    expression:
+//      start_time + i * period  [where i is a non-negative integer]
+//    During each period, the packet closest to the generated timestamp is
+//    emitted (latest in the case of ties).  In the first variant
+//    (sync_output_timestamps = true), the emitted packet is output at the
+//    generated timestamp.  In the second variant, the packet is output at
+//    its original timestamp.  Both variants emit exactly the same packets,
+//    but at different timestamps.
+//
+// Thinning period can be provided in the calculator options or via a
+// side packet with the tag "PERIOD".
+//
+// Example config:
+// node {
+//   calculator: "PacketThinnerCalculator"
+//   input_stream: "signal"
+//   output_stream: "output"
+//   options {
+//     [mediapipe.PacketThinnerCalculatorOptions.ext] {
+//       thinner_type: SYNC
+//       period: 10
+//       sync_output_timestamps: true
+//       update_frame_rate: false
+//     }
+//   }
+// }
+class PacketThinnerCalculator : public CalculatorBase {
+ public:
+  PacketThinnerCalculator() {}
+  ~PacketThinnerCalculator() override {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
+    if (cc->InputSidePackets().HasTag(kPeriodTag)) {
+      cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (cc->InputTimestamp() < start_time_) {
+      return ::mediapipe::OkStatus();  // Drop packets before start_time_.
+    } else if (cc->InputTimestamp() >= end_time_) {
+      if (!cc->Outputs().Index(0).IsClosed()) {
+        cc->Outputs()
+            .Index(0)
+            .Close();  // No more Packets will be output after end_time_.
+      }
+      return ::mediapipe::OkStatus();
+    } else {
+      return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC
+                 ? AsyncThinnerProcess(cc)
+                 : SyncThinnerProcess(cc);
+    }
+  }
+
+ private:
+  // Implementation of ASYNC and SYNC versions of thinner algorithm.
+  ::mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
+  ::mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
+
+  // Cached option.
+  PacketThinnerCalculatorOptions::ThinnerType thinner_type_;
+
+  // Given a Timestamp, finds the closest sync Timestamp
+  // based on start_time_ and period_.  This can be earlier or
+  // later than given Timestamp, but is guaranteed to be within
+  // half a period_.
+  Timestamp NearestSyncTimestamp(Timestamp now) const;
+
+  // Cached option used by both async and sync thinners.
+  TimestampDiff period_;  // Interval during which only one packet is emitted.
+  Timestamp start_time_;  // Cached option - default Timestamp::Min()
+  Timestamp end_time_;    // Cached option - default Timestamp::Max()
+
+  // Only used by async thinner:
+  Timestamp next_valid_timestamp_;  // Suppress packets until this timestamp.
+
+  // Only used by sync thinner:
+  Packet saved_packet_;          // Best packet not yet emitted.
+  bool sync_output_timestamps_;  // Cached option.
+};
+REGISTER_CALCULATOR(PacketThinnerCalculator);
+
+namespace {
+TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
+}  // namespace
+
+::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
+  auto& options = cc->Options<PacketThinnerCalculatorOptions>();
+
+  thinner_type_ = options.thinner_type();
+  // This check enables us to assume only two thinner types exist in Process()
+  CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
+        thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
+      << "Unsupported thinner type.";
+
+  if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+    // ASYNC thinner outputs packets with the same timestamp as their input so
+    // its safe to SetOffset(0). SYNC thinner manipulates timestamps of its
+    // output so we don't do this for that case.
+    cc->SetOffset(0);
+  }
+
+  if (cc->InputSidePackets().HasTag(kPeriodTag)) {
+    period_ =
+        TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get<int64>());
+  } else {
+    period_ = TimestampDiff(options.period());
+  }
+  CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
+
+  if (options.has_start_time()) {
+    start_time_ = Timestamp(options.start_time());
+  } else if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+    start_time_ = Timestamp::Min();
+  } else {
+    start_time_ = Timestamp(0);
+  }
+
+  end_time_ =
+      options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
+  CHECK_LT(start_time_, end_time_)
+      << "Invalid PacketThinner: start_time must be earlier than end_time";
+
+  sync_output_timestamps_ = options.sync_output_timestamps();
+
+  next_valid_timestamp_ = start_time_;
+  // Drop packets until this time.
+  cc->Outputs().Index(0).SetNextTimestampBound(start_time_);
+
+  if (!cc->Inputs().Index(0).Header().IsEmpty()) {
+    if (options.update_frame_rate()) {
+      const VideoHeader& video_header =
+          cc->Inputs().Index(0).Header().Get<VideoHeader>();
+      double new_frame_rate;
+      if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+        new_frame_rate =
+            video_header.frame_rate /
+            ceil(video_header.frame_rate * options.period() / kTimebaseUs);
+      } else {
+        const double sampling_rate = kTimebaseUs / options.period();
+        new_frame_rate = video_header.frame_rate < sampling_rate
+                             ? video_header.frame_rate
+                             : sampling_rate;
+      }
+      std::unique_ptr<VideoHeader> header(new VideoHeader);
+      header->format = video_header.format;
+      header->width = video_header.width;
+      header->height = video_header.height;
+      header->frame_rate = new_frame_rate;
+      cc->Outputs().Index(0).SetHeader(Adopt(header.release()));
+    } else {
+      cc->Outputs().Index(0).SetHeader(cc->Inputs().Index(0).Header());
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
+  // Emit any saved packets before quitting.
+  if (!saved_packet_.IsEmpty()) {
+    // Only sync thinner should have saved packets.
+    CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
+    if (sync_output_timestamps_) {
+      cc->Outputs().Index(0).AddPacket(
+          saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
+    } else {
+      cc->Outputs().Index(0).AddPacket(saved_packet_);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
+    CalculatorContext* cc) {
+  if (cc->InputTimestamp() >= next_valid_timestamp_) {
+    cc->Outputs().Index(0).AddPacket(
+        cc->Inputs().Index(0).Value());  // Emit current packet.
+    next_valid_timestamp_ = cc->InputTimestamp() + period_;
+    // Guaranteed not to emit packets seen during refractory period.
+    cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
+    CalculatorContext* cc) {
+  if (saved_packet_.IsEmpty()) {
+    // If no packet has been saved, store the current packet.
+    saved_packet_ = cc->Inputs().Index(0).Value();
+    cc->Outputs().Index(0).SetNextTimestampBound(
+        sync_output_timestamps_ ? NearestSyncTimestamp(cc->InputTimestamp())
+                                : cc->InputTimestamp());
+  } else {
+    // Saved packet exists -- update or emit.
+    const Timestamp saved = saved_packet_.Timestamp();
+    const Timestamp saved_sync = NearestSyncTimestamp(saved);
+    const Timestamp now = cc->InputTimestamp();
+    const Timestamp now_sync = NearestSyncTimestamp(now);
+    CHECK_LE(saved_sync, now_sync);
+    if (saved_sync == now_sync) {
+      // Saved Packet is in same interval as current packet.
+      // Replace saved packet with current if it is at least as
+      // central as the saved packet wrt temporal interval.
+      // [We break ties in favor of fresher packets]
+      if (abs(now - now_sync) <= abs(saved - saved_sync)) {
+        saved_packet_ = cc->Inputs().Index(0).Value();
+      }
+    } else {
+      // Saved packet is the best packet from earlier interval: emit!
+      if (sync_output_timestamps_) {
+        cc->Outputs().Index(0).AddPacket(saved_packet_.At(saved_sync));
+        cc->Outputs().Index(0).SetNextTimestampBound(now_sync);
+      } else {
+        cc->Outputs().Index(0).AddPacket(saved_packet_);
+        cc->Outputs().Index(0).SetNextTimestampBound(now);
+      }
+      // Current packet is the first one we've seen from new interval -- save!
+      saved_packet_ = cc->Inputs().Index(0).Value();
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
+  CHECK_NE(start_time_, Timestamp::Unset())
+      << "Method only valid for sync thinner calculator.";
+
+  // Computation is done using int64 arithmetic.  No easy way to avoid
+  // since Timestamps don't support div and multiply.
+  const int64 now64 = now.Value();
+  const int64 start64 = start_time_.Value();
+  const int64 period64 = period_.Value();
+  CHECK_LE(0, period64);
+
+  // Round now64 to its closest interval (units of period64).
+  int64 sync64 =
+      (now64 - start64 + period64 / 2) / period64 * period64 + start64;
+  CHECK_LE(abs(now64 - sync64), period64 / 2)
+      << "start64: " << start64 << "; now64: " << now64
+      << "; sync64: " << sync64;
+
+  return Timestamp(sync64);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_thinner_calculator.proto

@@ -0,0 +1,66 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message PacketThinnerCalculatorOptions {
+  extend CalculatorOptions {
+    optional PacketThinnerCalculatorOptions ext = 288533508;
+  }
+
+  enum ThinnerType {
+    ASYNC = 1;  // Asynchronous thinner, described below [default].
+    SYNC = 2;   // Synchronous thinner, also described below.
+  }
+  optional ThinnerType thinner_type = 1 [default = ASYNC];
+
+  // The period (in microsecond) specifies the temporal interval during which
+  // only a single packet is emitted in the output stream.  Has subtly different
+  // semantics depending on the thinner type, as follows.
+  //
+  // Async thinner: this option is a refractory period -- once a packet is
+  // emitted, we guarantee that no packets will be emitted for period ticks.
+  //
+  // Sync thinner: the period specifies a temporal interval during which
+  // only one packet is emitted.  The emitted packet is guaranteed to be
+  // the one closest to the center of the temporal interval (no guarantee on
+  // how ties are broken).  More specifically,
+  //   intervals are centered at start_time + i * period
+  //   (for non-negative integers i).
+  // Thus, each interval extends period/2 ticks before and after its center.
+  // Additionally, in the sync thinner any packets earlier than start_time
+  // are discarded and the thinner calls Close() once timestamp equals or
+  // exceeds end_time.
+  optional int64 period = 2 [default = 1];
+
+  // Packets before start_time and at/after end_time are discarded.
+  // Additionally, for a sync thinner, start time specifies the center of
+  // time invervals as described above and therefore should be set explicitly.
+  optional int64 start_time = 3;  // If not specified, set to 0 for SYNC type,
+                                  // and set to Timestamp::Min() for ASYNC type.
+  optional int64 end_time = 4;    // Set to Timestamp::Max() if not specified.
+
+  // Whether the timestamps of packets emitted by sync thinner should
+  // correspond to the center of their corresponding temporal interval.
+  // If false, packets emitted using original timestamp (as in async thinner).
+  optional bool sync_output_timestamps = 5 [default = true];
+
+  // If true, update the frame rate in the header, if it's available, to an
+  // estimated frame rate due to the sampling.
+  optional bool update_frame_rate = 6 [default = false];
+}

mediapipe/calculators/core/packet_thinner_calculator_test.cc

@@ -0,0 +1,357 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+// A simple version of CalculatorRunner with built-in convenience methods for
+// setting inputs from a vector and checking outputs against a vector of
+// expected outputs.
+class SimpleRunner : public CalculatorRunner {
+ public:
+  explicit SimpleRunner(const CalculatorOptions& options)
+      : CalculatorRunner("PacketThinnerCalculator", options) {
+    SetNumInputs(1);
+    SetNumOutputs(1);
+    SetNumInputSidePackets(0);
+  }
+
+  explicit SimpleRunner(const CalculatorGraphConfig::Node& node)
+      : CalculatorRunner(node) {}
+
+  void SetInput(const std::vector<int>& timestamp_list) {
+    MutableInputs()->Index(0).packets.clear();
+    for (const int ts : timestamp_list) {
+      MutableInputs()->Index(0).packets.push_back(
+          MakePacket<std::string>(absl::StrCat("Frame #", ts))
+              .At(Timestamp(ts)));
+    }
+  }
+
+  void SetFrameRate(const double frame_rate) {
+    auto video_header = absl::make_unique<VideoHeader>();
+    video_header->frame_rate = frame_rate;
+    MutableInputs()->Index(0).header = Adopt(video_header.release());
+  }
+
+  std::vector<int64> GetOutputTimestamps() const {
+    std::vector<int64> timestamps;
+    for (const Packet& packet : Outputs().Index(0).packets) {
+      timestamps.emplace_back(packet.Timestamp().Value());
+    }
+    return timestamps;
+  }
+
+  double GetFrameRate() const {
+    CHECK(!Outputs().Index(0).header.IsEmpty());
+    return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
+  }
+};
+
+// Check that thinner respects start_time and end_time options.
+// We only test with one thinner because the logic for start & end time
+// handling is shared across both types of thinner in Process().
+TEST(PacketThinnerCalculatorTest, StartAndEndTimeTest) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_start_time(4);
+  extension->set_end_time(12);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {5, 11};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, AsyncUniformStreamThinningTest) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("PacketThinnerCalculator");
+  node.add_input_side_packet("PERIOD:period");
+  node.add_input_stream("input_stream");
+  node.add_output_stream("output_stream");
+  auto* extension = node.mutable_options()->MutableExtension(
+      PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_start_time(0);
+  extension->set_sync_output_timestamps(false);
+
+  SimpleRunner runner(node);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest1) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("PacketThinnerCalculator");
+  node.add_input_side_packet("PERIOD:period");
+  node.add_input_stream("input_stream");
+  node.add_output_stream("output_stream");
+  auto* extension = node.mutable_options()->MutableExtension(
+      PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_sync_output_timestamps(false);
+
+  SimpleRunner runner(node);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest2) {
+  // Same test but now with synced timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+// Test: Given a stream with timestamps corresponding to first ten prime numbers
+// and period of 5, confirm whether timestamps of thinner stream matches
+// expectations.
+TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest1) {
+  // ASYNC thinner.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 7, 13, 19, 29};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest2) {
+  // SYNC with original timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 5, 11, 17, 19, 23, 29};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+// Confirm that Calculator correctly handles boundary cases.
+TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest1) {
+  // Odd period, negative start_time
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(-10);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  // Two timestamps falling on either side of a period boundary.
+  runner.SetInput({2, 3});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest2) {
+  // Even period, negative start_time, negative packet timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(-144);
+  extension->set_period(6);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  // Two timestamps falling on either side of a period boundary.
+  runner.SetInput({-4, -3, 8, 9});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {-6, 0, 6, 12};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest1) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true sampling period is 6.
+  EXPECT_DOUBLE_EQ(1000000.0 / 6, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest2) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({8, 16, 24, 32, 40, 48, 56});
+  runner.SetFrameRate(1000000.0 / 8);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<int64> expected_timestamps = {8, 16, 24, 32, 40, 48, 56};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true sampling period is still 8.
+  EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest3) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 5.
+  EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest4) {
+  // Same test but now with synced timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 5.
+  EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest5) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({8, 16, 24, 32, 40, 48, 56});
+  runner.SetFrameRate(1000000.0 / 8);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {10, 15, 25, 30, 40, 50, 55};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 8.
+  EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -17,6 +17,7 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
 
@@ -86,6 +87,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
         main_ts_.pop_front();
       }
     }
+    auto& loop_out = cc->Outputs().Get(loop_out_id_);
 
     while (!main_ts_.empty() && !loopback_packets_.empty()) {
       Timestamp main_timestamp = main_ts_.front();
@@ -95,18 +97,31 @@ class PreviousLoopbackCalculator : public CalculatorBase {
 
       if (previous_loopback.IsEmpty()) {
         // TODO: SetCompleteTimestampBound would be more useful.
-        cc->Outputs()
+        loop_out.SetNextTimestampBound(main_timestamp + 1);
-            .Get(loop_out_id_)
-            .SetNextTimestampBound(main_timestamp + 1);
       } else {
-        cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
+        loop_out.AddPacket(std::move(previous_loopback));
+      }
+    }
+
+    // In case of an empty loopback input, the next timestamp bound for
+    // loopback input is the loopback timestamp + 1.  The next timestamp bound
+    // for output is set and the main_ts_ vector is truncated accordingly.
+    if (loopback_packet.IsEmpty() &&
+        loopback_packet.Timestamp() != Timestamp::Unstarted()) {
+      Timestamp loopback_bound =
+          loopback_packet.Timestamp().NextAllowedInStream();
+      while (!main_ts_.empty() && main_ts_.front() <= loopback_bound) {
+        main_ts_.pop_front();
+      }
+      if (main_ts_.empty()) {
+        loop_out.SetNextTimestampBound(loopback_bound.NextAllowedInStream());
       }
     }
     if (!main_ts_.empty()) {
-      cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
+      loop_out.SetNextTimestampBound(main_ts_.front());
     }
     if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
-      cc->Outputs().Get(loop_out_id_).Close();
+      loop_out.Close();
     }
     return ::mediapipe::OkStatus();
   }

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -207,5 +207,64 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
+// Demonstrates that downstream calculators won't be blocked by
+// always-empty-LOOP-stream.
+TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
+  std::vector<Packet> outputs;
+  CalculatorGraphConfig graph_config_ =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: 'in'
+        node {
+          calculator: 'PreviousLoopbackCalculator'
+          input_stream: 'MAIN:in'
+          input_stream: 'LOOP:previous'
+          input_stream_info: { tag_index: 'LOOP' back_edge: true }
+          output_stream: 'PREV_LOOP:previous'
+        }
+        # This calculator synchronizes its inputs as normal, so it is used
+        # to check that both "in" and "previous" are ready.
+        node {
+          calculator: 'PassThroughCalculator'
+          input_stream: 'in'
+          input_stream: 'previous'
+          output_stream: 'out'
+          output_stream: 'previous2'
+        }
+        node {
+          calculator: 'PacketOnCloseCalculator'
+          input_stream: 'out'
+          output_stream: 'close_out'
+        }
+      )");
+  tool::AddVectorSink("close_out", &graph_config_, &outputs);
+
+  CalculatorGraph graph_;
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.StartRun({}));
+
+  auto send_packet = [&graph_](const std::string& input_name, int n) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        input_name, MakePacket<int>(n).At(Timestamp(n))));
+  };
+
+  send_packet("in", 0);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
+
+  for (int main_ts = 1; main_ts < 50; ++main_ts) {
+    send_packet("in", main_ts);
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+    std::vector<int64> ts_values = TimestampValues(outputs);
+    EXPECT_EQ(ts_values.size(), main_ts + 1);
+    for (int j = 0; j < main_ts; ++j) {
+      EXPECT_EQ(ts_values[j], j);
+    }
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
 }  // anonymous namespace
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -20,6 +20,10 @@
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
 namespace mediapipe {
 
 // Example config:
@@ -36,14 +40,21 @@ namespace mediapipe {
 //     }
 //   }
 // }
-typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
+typedef SplitVectorCalculator<TfLiteTensor, false>
+    SplitTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
 
-typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
+typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark, false>
     SplitLandmarkVectorCalculator;
 REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
 
-typedef SplitVectorCalculator<::mediapipe::NormalizedRect>
+typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
     SplitNormalizedRectVectorCalculator;
 REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
+    MovableSplitGlBufferVectorCalculator;
+REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator);
+#endif
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.h

@@ -15,12 +15,14 @@
 #ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
 
+#include <type_traits>
 #include <vector>
 
 #include "mediapipe/calculators/core/split_vector_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/interpreter.h"
@@ -29,6 +31,20 @@
 
 namespace mediapipe {
 
+template <typename T>
+using IsCopyable = std::enable_if_t<std::is_copy_constructible<T>::value, bool>;
+
+template <typename T>
+using IsNotCopyable =
+    std::enable_if_t<!std::is_copy_constructible<T>::value, bool>;
+
+template <typename T>
+using IsMovable = std::enable_if_t<std::is_move_constructible<T>::value, bool>;
+
+template <typename T>
+using IsNotMovable =
+    std::enable_if_t<!std::is_move_constructible<T>::value, bool>;
+
 // Splits an input packet with std::vector<T> into multiple std::vector<T>
 // output packets using the [begin, end) ranges specified in
 // SplitVectorCalculatorOptions. If the option "element_only" is set to true,
@@ -39,7 +55,7 @@ namespace mediapipe {
 // combined into one vector.
 // To use this class for a particular type T, register a calculator using
 // SplitVectorCalculator<T>.
-template <typename T>
+template <typename T, bool move_elements>
 class SplitVectorCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -51,23 +67,16 @@ class SplitVectorCalculator : public CalculatorBase {
     const auto& options =
         cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
 
+    if (!std::is_copy_constructible<T>::value || move_elements) {
+      // Ranges of elements shouldn't overlap when the vector contains
+      // non-copyable elements.
+      RET_CHECK_OK(checkRangesDontOverlap(options));
+    }
+
     if (options.combine_outputs()) {
       RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
       cc->Outputs().Index(0).Set<std::vector<T>>();
-      for (int i = 0; i < options.ranges_size() - 1; ++i) {
+      RET_CHECK_OK(checkRangesDontOverlap(options));
-        for (int j = i + 1; j < options.ranges_size(); ++j) {
-          const auto& range_0 = options.ranges(i);
-          const auto& range_1 = options.ranges(j);
-          if ((range_0.begin() >= range_1.begin() &&
-               range_0.begin() < range_1.end()) ||
-              (range_1.begin() >= range_0.begin() &&
-               range_1.begin() < range_0.end())) {
-            return ::mediapipe::InvalidArgumentError(
-                "Ranges must be non-overlapping when using combine_outputs "
-                "option.");
-          }
-        }
-      }
     } else {
       if (cc->Outputs().NumEntries() != options.ranges_size()) {
         return ::mediapipe::InvalidArgumentError(
@@ -117,14 +126,26 @@ class SplitVectorCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
-    const auto& input = cc->Inputs().Index(0).Get<std::vector<T>>();
+    if (cc->Inputs().Index(0).IsEmpty()) return ::mediapipe::OkStatus();
-    RET_CHECK_GE(input.size(), max_range_end_);
 
+    if (move_elements) {
+      return ProcessMovableElements<T>(cc);
+    } else {
+      return ProcessCopyableElements<T>(cc);
+    }
+  }
+
+  template <typename U, IsCopyable<U> = true>
+  ::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
+    // static_assert(std::is_copy_constructible<U>::value,
+    //              "Cannot copy non-copyable elements");
+    const auto& input = cc->Inputs().Index(0).Get<std::vector<U>>();
+    RET_CHECK_GE(input.size(), max_range_end_);
     if (combine_outputs_) {
-      auto output = absl::make_unique<std::vector<T>>();
+      auto output = absl::make_unique<std::vector<U>>();
       output->reserve(total_elements_);
       for (int i = 0; i < ranges_.size(); ++i) {
-        auto elements = absl::make_unique<std::vector<T>>(
+        auto elements = absl::make_unique<std::vector<U>>(
             input.begin() + ranges_[i].first,
             input.begin() + ranges_[i].second);
         output->insert(output->end(), elements->begin(), elements->end());
@@ -134,7 +155,7 @@ class SplitVectorCalculator : public CalculatorBase {
       if (element_only_) {
         for (int i = 0; i < ranges_.size(); ++i) {
           cc->Outputs().Index(i).AddPacket(
-              MakePacket<T>(input[ranges_[i].first]).At(cc->InputTimestamp()));
+              MakePacket<U>(input[ranges_[i].first]).At(cc->InputTimestamp()));
         }
       } else {
         for (int i = 0; i < ranges_.size(); ++i) {
@@ -149,7 +170,78 @@ class SplitVectorCalculator : public CalculatorBase {
     return ::mediapipe::OkStatus();
   }
 
+  template <typename U, IsNotCopyable<U> = true>
+  ::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
+    return ::mediapipe::InternalError("Cannot copy non-copyable elements.");
+  }
+
+  template <typename U, IsMovable<U> = true>
+  ::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
+    ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+        cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
+    if (!input_status.ok()) return input_status.status();
+    std::unique_ptr<std::vector<U>> input_vector =
+        std::move(input_status).ValueOrDie();
+    RET_CHECK_GE(input_vector->size(), max_range_end_);
+
+    if (combine_outputs_) {
+      auto output = absl::make_unique<std::vector<U>>();
+      output->reserve(total_elements_);
+      for (int i = 0; i < ranges_.size(); ++i) {
+        output->insert(
+            output->end(),
+            std::make_move_iterator(input_vector->begin() + ranges_[i].first),
+            std::make_move_iterator(input_vector->begin() + ranges_[i].second));
+      }
+      cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<U>(std::move(input_vector->at(ranges_[i].first)))
+                  .At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          auto output = absl::make_unique<std::vector<T>>();
+          output->insert(
+              output->end(),
+              std::make_move_iterator(input_vector->begin() + ranges_[i].first),
+              std::make_move_iterator(input_vector->begin() +
+                                      ranges_[i].second));
+          cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U, IsNotMovable<U> = true>
+  ::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
+    return ::mediapipe::InternalError("Cannot move non-movable elements.");
+  }
+
  private:
+  static ::mediapipe::Status checkRangesDontOverlap(
+      const ::mediapipe::SplitVectorCalculatorOptions& options) {
+    for (int i = 0; i < options.ranges_size() - 1; ++i) {
+      for (int j = i + 1; j < options.ranges_size(); ++j) {
+        const auto& range_0 = options.ranges(i);
+        const auto& range_1 = options.ranges(j);
+        if ((range_0.begin() >= range_1.begin() &&
+             range_0.begin() < range_1.end()) ||
+            (range_1.begin() >= range_0.begin() &&
+             range_1.begin() < range_0.end())) {
+          return ::mediapipe::InvalidArgumentError(
+              "Ranges must be non-overlapping when using combine_outputs "
+              "option.");
+        }
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
   std::vector<std::pair<int32, int32>> ranges_;
   int32 max_range_end_ = -1;
   int32 total_elements_ = 0;

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -452,4 +452,243 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   ASSERT_FALSE(graph.Initialize(graph_config).ok());
 }
 
+typedef SplitVectorCalculator<std::unique_ptr<int>, true>
+    MovableSplitUniqueIntPtrCalculator;
+REGISTER_CALCULATOR(MovableSplitUniqueIntPtrCalculator);
+
+class MovableSplitUniqueIntPtrCalculatorTest : public ::testing::Test {
+ protected:
+  void ValidateVectorOutput(std::vector<Packet>& output_packets,
+                            int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const std::vector<std::unique_ptr<int>>& output_vec =
+        output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
+    ASSERT_EQ(expected_elements, output_vec.size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const int expected_value = input_begin_index + i;
+      const std::unique_ptr<int>& result = output_vec[i];
+      ASSERT_NE(result, nullptr);
+      ASSERT_EQ(expected_value, *result);
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int expected_value) {
+    ASSERT_EQ(1, output_packets.size());
+    const std::unique_ptr<int>& result =
+        output_packets[0].Get<std::unique_ptr<int>>();
+    ASSERT_NE(result, nullptr);
+    ASSERT_EQ(expected_value, *result);
+  }
+
+  void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
+                                    int expected_elements,
+                                    std::vector<int>& input_begin_indices,
+                                    std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const std::vector<std::unique_ptr<int>>& output_vector =
+        output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
+    ASSERT_EQ(expected_elements, output_vector.size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const int expected_value = i;
+        const std::unique_ptr<int>& result = output_vector[element_id];
+        ASSERT_NE(result, nullptr);
+        ASSERT_EQ(expected_value, *result);
+        ++element_id;
+      }
+    }
+  }
+};
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/0);
+  ValidateVectorOutput(range_1_packets, /*expected_elements=*/3,
+                       /*input_begin_index=*/1);
+  ValidateVectorOutput(range_2_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/4);
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ValidateElementOutput(range_0_packets, /*expected_value=*/0);
+  ValidateElementOutput(range_1_packets, /*expected_value=*/2);
+  ValidateElementOutput(range_2_packets, /*expected_value=*/4);
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
+                               input_begin_indices, input_end_indices);
+}
+
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -80,7 +80,9 @@ mediapipe_cc_proto_library(
     name = "opencv_image_encoder_calculator_cc_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//visibility:public",
+    ],
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -501,8 +501,11 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
     row_max = std::max(row_max, transformed_points_[i * 2 + 1]);
   }
 
-  *dst_width = std::round((col_max - col_min) * src_width);
+  int width = static_cast<int>(std::round((col_max - col_min) * src_width));
-  *dst_height = std::round((row_max - row_min) * src_height);
+  int height = static_cast<int>(std::round((row_max - row_min) * src_height));
+  // Minimum output dimension 1x1 prevents creation of textures with 0x0.
+  *dst_width = std::max(1, width);
+  *dst_height = std::max(1, height);
 }
 
 }  // namespace mediapipe

mediapipe/calculators/image/scale_image_calculator.cc

@@ -474,13 +474,20 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
           input_width_, "x", input_height_));
     }
     if (input_format_ != image_frame.Format()) {
+      std::string image_frame_format_desc, input_format_desc;
+#ifdef MEDIAPIPE_MOBILE
+      image_frame_format_desc = std::to_string(image_frame.Format());
+      input_format_desc = std::to_string(input_format_);
+#else
       const proto_ns::EnumDescriptor* desc = ImageFormat::Format_descriptor();
+      image_frame_format_desc =
+          desc->FindValueByNumber(image_frame.Format())->DebugString();
+      input_format_desc = desc->FindValueByNumber(input_format_)->DebugString();
+#endif  // MEDIAPIPE_MOBILE
       return tool::StatusFail(absl::StrCat(
           "If a header specifies a format, then image frames on "
           "the stream must have that format.  Actual format ",
-          desc->FindValueByNumber(image_frame.Format())->DebugString(),
+          image_frame_format_desc, " but expected ", input_format_desc));
-          " but expected ",
-          desc->FindValueByNumber(input_format_)->DebugString()));
     }
   }
   return ::mediapipe::OkStatus();

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -264,7 +264,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
     if (options.output_only_if_all_present()) {
       ::mediapipe::Status status = VerifySequence();
       if (!status.ok()) {
-        cc->GetCounter(status.error_message())->Increment();
+        cc->GetCounter(status.ToString())->Increment();
         return status;
       }
     }

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -454,7 +454,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
 
     // RET_CHECK on the tf::Status object itself in order to print an
     // informative error message.
-    RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
 
     const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
     cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -109,7 +109,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
 
     RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
     const tf::Status tf_status = session->session->Create(graph_def);
-    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString();
 
     for (const auto& key_value : options.tag_to_tensor_names()) {
       session->tag_to_tensor_map[key_value.first] = key_value.second;
@@ -119,7 +119,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
           session->session->Run({}, {}, initialization_op_names, {});
       // RET_CHECK on the tf::Status object itself in order to print an
       // informative error message.
-      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
     cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc

@@ -109,7 +109,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
 
     RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
     const tf::Status tf_status = session->session->Create(graph_def);
-    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString();
 
     for (const auto& key_value : options.tag_to_tensor_names()) {
       session->tag_to_tensor_map[key_value.first] = key_value.second;
@@ -119,7 +119,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
           session->session->Run({}, {}, initialization_op_names, {});
       // RET_CHECK on the tf::Status object itself in order to print an
       // informative error message.
-      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
     output_side_packets->Tag("SESSION") = Adopt(session.release());

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -140,7 +140,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
     if (!status.ok()) {
       return ::mediapipe::Status(
           static_cast<::mediapipe::StatusCode>(status.code()),
-          status.error_message());
+          status.ToString());
     }
 
     auto session = absl::make_unique<TensorFlowSession>();

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -135,7 +135,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
     if (!status.ok()) {
       return ::mediapipe::Status(
           static_cast<::mediapipe::StatusCode>(status.code()),
-          status.error_message());
+          status.ToString());
     }
 
     auto session = absl::make_unique<TensorFlowSession>();

mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc

@@ -81,7 +81,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
   auto tf_status = tensorflow::Env::Default()->NewRandomAccessFile(
       cc->InputSidePackets().Tag(kTFRecordPath).Get<std::string>(), &file);
   RET_CHECK(tf_status.ok())
-      << "Failed to open tfrecord file: " << tf_status.error_message();
+      << "Failed to open tfrecord file: " << tf_status.ToString();
   tensorflow::io::RecordReader reader(file.get(),
                                       tensorflow::io::RecordReaderOptions());
   tensorflow::uint64 offset = 0;
@@ -94,7 +94,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
   while (current_idx <= target_idx) {
     tf_status = reader.ReadRecord(&offset, &example_str);
     RET_CHECK(tf_status.ok())
-        << "Failed to read tfrecord: " << tf_status.error_message();
+        << "Failed to read tfrecord: " << tf_status.ToString();
     if (current_idx == target_idx) {
       if (cc->OutputSidePackets().HasTag(kExampleTag)) {
         tensorflow::Example tf_example;

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -294,11 +294,15 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
       if (use_quantized_tensors_) {
         RET_CHECK(image_frame.Format() != mediapipe::ImageFormat::VEC32F1)
             << "Only 8-bit input images are supported for quantization.";
+        quant.type = kTfLiteAffineQuantization;
+        quant.params = nullptr;
         // Optional: Set 'quant' quantization params here if needed.
         interpreter_->SetTensorParametersReadWrite(0, kTfLiteUInt8, "",
                                                    {channels_preserved}, quant);
       } else {
-        // Default TfLiteQuantization used for no quantization.
+        // Initialize structure for no quantization.
+        quant.type = kTfLiteNoQuantization;
+        quant.params = nullptr;
         interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "",
                                                    {channels_preserved}, quant);
       }
@@ -422,40 +426,35 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 #elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
-  {
+  id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
+
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+  id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
-    command_buffer.label = @"TfLiteConverterCalculatorConvert";
+  command_buffer.label = @"TfLiteConverterCalculatorConvertAndBlit";
-    id<MTLComputeCommandEncoder> compute_encoder =
+  id<MTLComputeCommandEncoder> compute_encoder =
-        [command_buffer computeCommandEncoder];
+      [command_buffer computeCommandEncoder];
-    [compute_encoder setComputePipelineState:gpu_data_out_->pipeline_state];
+  [compute_encoder setComputePipelineState:gpu_data_out_->pipeline_state];
-    [compute_encoder setTexture:src_texture atIndex:0];
+  [compute_encoder setTexture:src_texture atIndex:0];
-    [compute_encoder setBuffer:gpu_data_out_->buffer offset:0 atIndex:1];
+  [compute_encoder setBuffer:gpu_data_out_->buffer offset:0 atIndex:1];
-    MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
+  MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
-    MTLSize threadgroups =
+  MTLSize threadgroups =
-        MTLSizeMake(NumGroups(input.width(), kWorkgroupSize),
+      MTLSizeMake(NumGroups(input.width(), kWorkgroupSize),
-                    NumGroups(input.height(), kWorkgroupSize), 1);
+                  NumGroups(input.height(), kWorkgroupSize), 1);
-    [compute_encoder dispatchThreadgroups:threadgroups
+  [compute_encoder dispatchThreadgroups:threadgroups
-                    threadsPerThreadgroup:threads_per_group];
+                  threadsPerThreadgroup:threads_per_group];
-    [compute_encoder endEncoding];
+  [compute_encoder endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
-  }
 
   // Copy into outputs.
   // TODO Avoid this copy.
   auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
   output_tensors->resize(1);
-  {
+  id<MTLDevice> device = gpu_helper_.mtlDevice;
-    id<MTLDevice> device = gpu_helper_.mtlDevice;
+  output_tensors->at(0) =
-    output_tensors->at(0) =
+      [device newBufferWithLength:gpu_data_out_->elements * sizeof(float)
-        [device newBufferWithLength:gpu_data_out_->elements * sizeof(float)
+                          options:MTLResourceStorageModeShared];
-                            options:MTLResourceStorageModeShared];
+  [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0)
-    [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0)
+                             from:gpu_data_out_->buffer
-                               from:gpu_data_out_->buffer
+                         blocking:false
-                           blocking:true
+                    commandBuffer:command_buffer];
-                      commandBuffer:[gpu_helper_ commandBuffer]];
-  }
 
   cc->Outputs()
       .Tag("TENSORS_GPU")

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -56,6 +56,10 @@
 #endif  // ANDROID
 
 namespace {
+// Commonly used to compute the number of blocks to launch in a kernel.
+int NumGroups(const int size, const int group_size) {  // NOLINT
+  return (size + group_size - 1) / group_size;
+}
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
@@ -176,12 +180,13 @@ class TfLiteInferenceCalculator : public CalculatorBase {
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
-  std::unique_ptr<GPUData> gpu_data_in_;
+  std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
 #elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
-  std::unique_ptr<GPUData> gpu_data_in_;
+  std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  id<MTLComputePipelineState> fp32_to_fp16_program_;
   TFLBufferConvert* converter_from_BPHWC4_ = nil;
 #endif
 
@@ -308,22 +313,41 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
-    RET_CHECK_EQ(input_tensors.size(), 1);
+    RET_CHECK_GT(input_tensors.size(), 0);
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
           // Explicit copy input.
-          RET_CHECK_CALL(CopyBuffer(input_tensors[0], gpu_data_in_->buffer));
+          gpu_data_in_.resize(input_tensors.size());
+          for (int i = 0; i < input_tensors.size(); ++i) {
+            RET_CHECK_CALL(
+                CopyBuffer(input_tensors[i], gpu_data_in_[i]->buffer));
+          }
+
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
-    RET_CHECK_EQ(input_tensors.size(), 1);
+    RET_CHECK_GT(input_tensors.size(), 0);
-    // Explicit copy input.
+    // Explicit copy input with conversion float 32 bits to 16 bits.
-    [MPPMetalUtil blitMetalBufferTo:gpu_data_in_->buffer
+    gpu_data_in_.resize(input_tensors.size());
-                               from:input_tensors[0]
+    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-                           blocking:true
+    command_buffer.label = @"TfLiteInferenceCalculatorConvert";
-                      commandBuffer:[gpu_helper_ commandBuffer]];
+    id<MTLComputeCommandEncoder> compute_encoder =
+        [command_buffer computeCommandEncoder];
+    [compute_encoder setComputePipelineState:fp32_to_fp16_program_];
+    for (int i = 0; i < input_tensors.size(); ++i) {
+      [compute_encoder setBuffer:input_tensors[i] offset:0 atIndex:0];
+      [compute_encoder setBuffer:gpu_data_in_[i]->buffer offset:0 atIndex:1];
+      constexpr int kWorkgroupSize = 64;  // Block size for GPU shader.
+      MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, 1, 1);
+      const int threadgroups =
+          NumGroups(gpu_data_in_[i]->elements, kWorkgroupSize);
+      [compute_encoder dispatchThreadgroups:MTLSizeMake(threadgroups, 1, 1)
+                      threadsPerThreadgroup:threads_per_group];
+    }
+    [compute_encoder endEncoding];
+    [command_buffer commit];
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif
@@ -404,7 +428,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     }
     [convert_command endEncoding];
     [command_buffer commit];
-    [command_buffer waitUntilCompleted];
     cc->Outputs()
         .Tag("TENSORS_GPU")
         .Add(output_tensors.release(), cc->InputTimestamp());
@@ -432,7 +455,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
         TfLiteGpuDelegateDelete(delegate_);
-        gpu_data_in_.reset();
+        for (int i = 0; i < gpu_data_in_.size(); ++i) {
+          gpu_data_in_[i].reset();
+        }
         for (int i = 0; i < gpu_data_out_.size(); ++i) {
           gpu_data_out_[i].reset();
         }
@@ -440,7 +465,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       }));
 #elif defined(MEDIAPIPE_IOS)
       TFLGpuDelegateDelete(delegate_);
-      gpu_data_in_.reset();
+      for (int i = 0; i < gpu_data_in_.size(); ++i) {
+        gpu_data_in_[i].reset();
+      }
       for (int i = 0; i < gpu_data_out_.size(); ++i) {
         gpu_data_out_[i].reset();
       }
@@ -464,7 +491,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // Get model name.
   if (!options.model_path().empty()) {
-    auto model_path = options.model_path();
+    std::string model_path = options.model_path();
 
     ASSIGN_OR_RETURN(model_path_, mediapipe::PathToResourceAsFile(model_path));
   } else {
@@ -545,24 +572,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   if (gpu_input_) {
     // Get input image sizes.
-    gpu_data_in_ = absl::make_unique<GPUData>();
     const auto& input_indices = interpreter_->inputs();
-    RET_CHECK_EQ(input_indices.size(), 1);  // TODO accept > 1.
+    gpu_data_in_.resize(input_indices.size());
-    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+    for (int i = 0; i < input_indices.size(); ++i) {
-    gpu_data_in_->elements = 1;
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
-    for (int d = 0; d < tensor->dims->size; ++d) {
+      gpu_data_in_[i] = absl::make_unique<GPUData>();
-      gpu_data_in_->elements *= tensor->dims->data[d];
+      gpu_data_in_[i]->elements = 1;
+      for (int d = 0; d < tensor->dims->size; ++d) {
+        gpu_data_in_[i]->elements *= tensor->dims->data[d];
+      }
+      // Create and bind input buffer.
+      RET_CHECK_CALL(
+          ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+              gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
+      RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
+                       delegate_, gpu_data_in_[i]->buffer.id(),
+                       interpreter_->inputs()[i]),
+                   kTfLiteOk);
     }
-    CHECK_GE(tensor->dims->data[3], 1);
-    CHECK_LE(tensor->dims->data[3], 4);
-    CHECK_NE(tensor->dims->data[3], 2);
-    // Create and bind input buffer.
-    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
-        gpu_data_in_->elements, &gpu_data_in_->buffer));
-    RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
-                     delegate_, gpu_data_in_->buffer.id(),
-                     interpreter_->inputs()[0]),  // First tensor only
-                 kTfLiteOk);
   }
   if (gpu_output_) {
     // Get output image sizes.
@@ -594,41 +621,68 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif  // OpenGL
 
 #if defined(MEDIAPIPE_IOS)
+  const int kHalfSize = 2;  // sizeof(half)
   // Configure and create the delegate.
   TFLGpuDelegateOptions options;
-  options.allow_precision_loss = false;  // Must match converter, F=float/T=half
+  options.allow_precision_loss = true;
   options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
   if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   if (gpu_input_) {
     // Get input image sizes.
-    gpu_data_in_ = absl::make_unique<GPUData>();
     const auto& input_indices = interpreter_->inputs();
-    RET_CHECK_EQ(input_indices.size(), 1);
+    gpu_data_in_.resize(input_indices.size());
-    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+    for (int i = 0; i < input_indices.size(); ++i) {
-    gpu_data_in_->elements = 1;
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
-    // On iOS GPU, input must be 4 channels, regardless of what model expects.
+      gpu_data_in_[i] = absl::make_unique<GPUData>();
-    {
+      gpu_data_in_[i]->shape.b = tensor->dims->data[0];
-      gpu_data_in_->elements *= tensor->dims->data[0];  // batch
+      gpu_data_in_[i]->shape.h = tensor->dims->data[1];
-      gpu_data_in_->elements *= tensor->dims->data[1];  // height
+      gpu_data_in_[i]->shape.w = tensor->dims->data[2];
-      gpu_data_in_->elements *= tensor->dims->data[2];  // width
+      // On iOS GPU, input must be 4 channels, regardless of what model expects.
-      gpu_data_in_->elements *= 4;                      // channels
+      gpu_data_in_[i]->shape.c = 4;
+      gpu_data_in_[i]->elements =
+          gpu_data_in_[i]->shape.b * gpu_data_in_[i]->shape.h *
+          gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c;
+      // Input to model can be RGBA only.
+      if (tensor->dims->data[3] != 4) {
+        LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
+      }
+      const std::string shader_source =
+          absl::Substitute(R"(#include <metal_stdlib>
+        using namespace metal;
+        kernel void convertKernel(device float4* const input_buffer [[buffer(0)]],
+                                  device half4* output_buffer [[buffer(1)]],
+                                  uint gid [[thread_position_in_grid]]) {
+          if (gid >= $0) return;
+          output_buffer[gid] = half4(input_buffer[gid]);
+        })",
+                           gpu_data_in_[i]->elements / 4);
+      NSString* library_source =
+          [NSString stringWithUTF8String:shader_source.c_str()];
+      NSError* error = nil;
+      id<MTLLibrary> library =
+          [device newLibraryWithSource:library_source options:nil error:&error];
+      RET_CHECK(library != nil) << "Couldn't create shader library "
+                                << [[error localizedDescription] UTF8String];
+      id<MTLFunction> kernel_func = nil;
+      kernel_func = [library newFunctionWithName:@"convertKernel"];
+      RET_CHECK(kernel_func != nil) << "Couldn't create kernel function.";
+      fp32_to_fp16_program_ =
+          [device newComputePipelineStateWithFunction:kernel_func error:&error];
+      RET_CHECK(fp32_to_fp16_program_ != nil)
+          << "Couldn't create pipeline state "
+          << [[error localizedDescription] UTF8String];
+
+      // Create and bind input buffer.
+      gpu_data_in_[i]->buffer =
+          [device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
+                              options:MTLResourceStorageModeShared];
+      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+      RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
+                       delegate_, input_indices[i], gpu_data_in_[i]->buffer),
+                   true);
     }
-    // Input to model can be RGBA only.
-    if (tensor->dims->data[3] != 4) {
-      LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
-    }
-    // Create and bind input buffer.
-    gpu_data_in_->buffer =
-        [device newBufferWithLength:gpu_data_in_->elements * sizeof(float)
-                            options:MTLResourceStorageModeShared];
-    RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
-    RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
-                     delegate_,
-                     input_indices[0],  // First tensor only
-                     gpu_data_in_->buffer),
-                 true);
   }
   if (gpu_output_) {
     // Get output image sizes.
@@ -669,15 +723,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     interpreter_->SetAllowBufferHandleOutput(true);
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
       gpu_data_out_[i]->buffer =
-          [device newBufferWithLength:gpu_data_out_[i]->elements * sizeof(float)
+          [device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
                               options:MTLResourceStorageModeShared];
       RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                        delegate_, output_indices[i], gpu_data_out_[i]->buffer),
                    true);
     }
+
     // Create converter for GPU output.
     converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
-                                                            isFloat16:false
+                                                            isFloat16:true
                                                       convertToPBHWC4:false];
     if (converter_from_BPHWC4_ == nil) {
       return mediapipe::InternalError(

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -472,11 +472,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
   // Copy inputs.
   [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_boxes_buffer
                              from:input_tensors[0]
-                         blocking:true
+                         blocking:false
                     commandBuffer:[gpu_helper_ commandBuffer]];
   [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_scores_buffer
                              from:input_tensors[1]
-                         blocking:true
+                         blocking:false
                     commandBuffer:[gpu_helper_ commandBuffer]];
   if (!anchors_init_) {
     if (side_packet_anchors_) {
@@ -491,48 +491,37 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
       RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors);
       [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_anchors_buffer
                                  from:input_tensors[2]
-                             blocking:true
+                             blocking:false
                         commandBuffer:[gpu_helper_ commandBuffer]];
     }
     anchors_init_ = true;
   }
 
   // Run shaders.
-  {
+  id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+  command_buffer.label = @"TfLiteDecodeAndScoreBoxes";
-    command_buffer.label = @"TfLiteDecodeBoxes";
+  id<MTLComputeCommandEncoder> command_encoder =
-    id<MTLComputeCommandEncoder> decode_command =
+      [command_buffer computeCommandEncoder];
-        [command_buffer computeCommandEncoder];
+  [command_encoder setComputePipelineState:gpu_data_->decode_program];
-    [decode_command setComputePipelineState:gpu_data_->decode_program];
+  [command_encoder setBuffer:gpu_data_->decoded_boxes_buffer
-    [decode_command setBuffer:gpu_data_->decoded_boxes_buffer
+                      offset:0
-                       offset:0
+                     atIndex:0];
-                      atIndex:0];
+  [command_encoder setBuffer:gpu_data_->raw_boxes_buffer offset:0 atIndex:1];
-    [decode_command setBuffer:gpu_data_->raw_boxes_buffer offset:0 atIndex:1];
+  [command_encoder setBuffer:gpu_data_->raw_anchors_buffer offset:0 atIndex:2];
-    [decode_command setBuffer:gpu_data_->raw_anchors_buffer offset:0 atIndex:2];
+  MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
-    MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
+  MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
-    MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
+  [command_encoder dispatchThreadgroups:decode_threadgroups
-    [decode_command dispatchThreadgroups:decode_threadgroups
+                  threadsPerThreadgroup:decode_threads_per_group];
-                   threadsPerThreadgroup:decode_threads_per_group];
+
-    [decode_command endEncoding];
+  [command_encoder setComputePipelineState:gpu_data_->score_program];
-    [command_buffer commit];
+  [command_encoder setBuffer:gpu_data_->scored_boxes_buffer offset:0 atIndex:0];
-    [command_buffer waitUntilCompleted];
+  [command_encoder setBuffer:gpu_data_->raw_scores_buffer offset:0 atIndex:1];
-  }
+  MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
-  {
+  MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+  [command_encoder dispatchThreadgroups:score_threadgroups
-    command_buffer.label = @"TfLiteScoreBoxes";
-    id<MTLComputeCommandEncoder> score_command =
-        [command_buffer computeCommandEncoder];
-    [score_command setComputePipelineState:gpu_data_->score_program];
-    [score_command setBuffer:gpu_data_->scored_boxes_buffer offset:0 atIndex:0];
-    [score_command setBuffer:gpu_data_->raw_scores_buffer offset:0 atIndex:1];
-    MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
-    MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
-    [score_command dispatchThreadgroups:score_threadgroups
                   threadsPerThreadgroup:score_threads_per_group];
-    [score_command endEncoding];
+  [command_encoder endEncoding];
-    [command_buffer commit];
+  [MPPMetalUtil commitCommandBufferAndWait:command_buffer];
-    [command_buffer waitUntilCompleted];
-  }
 
   // Copy decoded boxes from GPU to CPU.
   std::vector<float> boxes(num_boxes_ * num_coords_);

mediapipe/calculators/util/BUILD

@@ -983,3 +983,31 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
     ],
 )
+
+cc_library(
+    name = "detections_to_timed_box_list_calculator",
+    srcs = ["detections_to_timed_box_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:box_tracker",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "detection_unique_id_calculator",
+    srcs = ["detection_unique_id_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/util/detection_unique_id_calculator.cc

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+
+// Each detection processed by DetectionUniqueIDCalculator will be assigned an
+// unique id that starts from 1. If a detection already has an ID other than 0,
+// the ID will be overwritten.
+static int64 detection_id = 0;
+
+inline int GetNextDetectionId() { return ++detection_id; }
+
+}  // namespace
+
+// Assign a unique id to detections.
+// Note that the calculator will consume the input vector of Detection or
+// DetectionList. So the input stream can not be connected to other calculators.
+//
+// Example config:
+// node {
+//   calculator: "DetectionUniqueIdCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "DETECTIONS:output_detections"
+// }
+class DetectionUniqueIdCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
+              cc->Inputs().HasTag(kDetectionsTag))
+        << "None of the input streams are provided.";
+
+    if (cc->Inputs().HasTag(kDetectionListTag)) {
+      RET_CHECK(cc->Outputs().HasTag(kDetectionListTag));
+      cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+      cc->Outputs().Tag(kDetectionListTag).Set<DetectionList>();
+    }
+    if (cc->Inputs().HasTag(kDetectionsTag)) {
+      RET_CHECK(cc->Outputs().HasTag(kDetectionsTag));
+      cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+      cc->Outputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(::mediapipe::TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(DetectionUniqueIdCalculator);
+
+::mediapipe::Status DetectionUniqueIdCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kDetectionListTag) &&
+      !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) {
+    auto result =
+        cc->Inputs().Tag(kDetectionListTag).Value().Consume<DetectionList>();
+    if (result.ok()) {
+      auto detection_list = std::move(result).ValueOrDie();
+      for (Detection& detection : *detection_list->mutable_detection()) {
+        detection.set_detection_id(GetNextDetectionId());
+      }
+      cc->Outputs()
+          .Tag(kDetectionListTag)
+          .Add(detection_list.release(), cc->InputTimestamp());
+    }
+  }
+
+  if (cc->Inputs().HasTag(kDetectionsTag) &&
+      !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) {
+    auto result = cc->Inputs()
+                      .Tag(kDetectionsTag)
+                      .Value()
+                      .Consume<std::vector<Detection>>();
+    if (result.ok()) {
+      auto detections = std::move(result).ValueOrDie();
+      for (Detection& detection : *detections) {
+        detection.set_detection_id(GetNextDetectionId());
+      }
+      cc->Outputs()
+          .Tag(kDetectionsTag)
+          .Add(detections.release(), cc->InputTimestamp());
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/detections_to_render_data_calculator.cc

@@ -39,6 +39,8 @@ constexpr char kKeypointLabel[] = "KEYPOINT";
 // The ratio of detection label font height to the height of detection bounding
 // box.
 constexpr double kLabelToBoundingBoxRatio = 0.1;
+// Perserve 2 decimal digits.
+constexpr float kNumScoreDecimalDigitsMultipler = 100;
 
 }  // namespace
 
@@ -235,18 +237,26 @@ void DetectionsToRenderDataCalculator::AddLabels(
     std::string label_str = detection.label().empty()
                                 ? absl::StrCat(detection.label_id(i))
                                 : detection.label(i);
+    const float rounded_score =
+        std::round(detection.score(i) * kNumScoreDecimalDigitsMultipler) /
+        kNumScoreDecimalDigitsMultipler;
     std::string label_and_score =
-        absl::StrCat(label_str, options.text_delimiter(), detection.score(i),
+        absl::StrCat(label_str, options.text_delimiter(), rounded_score,
                      options.text_delimiter());
     label_and_scores.push_back(label_and_score);
   }
   std::vector<std::string> labels;
+  if (options.render_detection_id()) {
+    const std::string detection_id_str =
+        absl::StrCat("Id: ", detection.detection_id());
+    labels.push_back(detection_id_str);
+  }
   if (options.one_label_per_line()) {
-    labels.swap(label_and_scores);
+    labels.insert(labels.end(), label_and_scores.begin(),
+                  label_and_scores.end());
   } else {
     labels.push_back(absl::StrJoin(label_and_scores, ""));
   }
-
   // Add the render annotations for "label(_id),score".
   for (int i = 0; i < labels.size(); ++i) {
     auto label = labels.at(i);

mediapipe/calculators/util/detections_to_render_data_calculator.proto

@@ -53,4 +53,7 @@ message DetectionsToRenderDataCalculatorOptions {
   // instances of this calculator are present in the graph, this value
   // should be unique among them.
   optional string scene_class = 7 [default = "DETECTION"];
+
+  // If true, renders the detection id in the first line before the labels.
+  optional bool render_detection_id = 8 [default = false];
 }

mediapipe/calculators/util/detections_to_timed_box_list_calculator.cc

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+constexpr char kBoxesTag[] = "BOXES";
+
+}  // namespace
+
+// A calculator that converts Detection proto to TimedBoxList proto for
+// tracking.
+//
+// Please note that only Location Data formats of RELATIVE_BOUNDING_BOX are
+// supported.
+//
+// Example config:
+// node {
+//   calculator: "DetectionsToTimedBoxListCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "BOXES:boxes"
+// }
+class DetectionsToTimedBoxListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
+              cc->Inputs().HasTag(kDetectionsTag))
+        << "None of the input streams are provided.";
+    if (cc->Inputs().HasTag(kDetectionListTag)) {
+      cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+    }
+    if (cc->Inputs().HasTag(kDetectionsTag)) {
+      cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+    }
+    cc->Outputs().Tag(kBoxesTag).Set<TimedBoxProtoList>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void ConvertDetectionToTimedBox(const Detection& detection,
+                                  TimedBoxProto* box, CalculatorContext* cc);
+};
+REGISTER_CALCULATOR(DetectionsToTimedBoxListCalculator);
+
+::mediapipe::Status DetectionsToTimedBoxListCalculator::Process(
+    CalculatorContext* cc) {
+  auto output_timed_box_list = absl::make_unique<TimedBoxProtoList>();
+
+  if (cc->Inputs().HasTag(kDetectionListTag)) {
+    const auto& detection_list =
+        cc->Inputs().Tag(kDetectionListTag).Get<DetectionList>();
+    for (const auto& detection : detection_list.detection()) {
+      TimedBoxProto* box = output_timed_box_list->add_box();
+      ConvertDetectionToTimedBox(detection, box, cc);
+    }
+  }
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    const auto& detections =
+        cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>();
+    for (const auto& detection : detections) {
+      TimedBoxProto* box = output_timed_box_list->add_box();
+      ConvertDetectionToTimedBox(detection, box, cc);
+    }
+  }
+
+  cc->Outputs().Tag(kBoxesTag).Add(output_timed_box_list.release(),
+                                   cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+void DetectionsToTimedBoxListCalculator::ConvertDetectionToTimedBox(
+    const Detection& detection, TimedBoxProto* box, CalculatorContext* cc) {
+  const auto& relative_bounding_box =
+      detection.location_data().relative_bounding_box();
+  box->set_left(relative_bounding_box.xmin());
+  box->set_right(relative_bounding_box.xmin() + relative_bounding_box.width());
+  box->set_top(relative_bounding_box.ymin());
+  box->set_bottom(relative_bounding_box.ymin() +
+                  relative_bounding_box.height());
+  box->set_id(detection.detection_id());
+  box->set_time_msec(cc->InputTimestamp().Microseconds() / 1000);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/BUILD

@@ -37,6 +37,86 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "motion_analysis_calculator_proto",
+    srcs = ["motion_analysis_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:motion_analysis_proto",
+    ],
+)
+
+proto_library(
+    name = "flow_packager_calculator_proto",
+    srcs = ["flow_packager_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:flow_packager_proto",
+    ],
+)
+
+proto_library(
+    name = "box_tracker_calculator_proto",
+    srcs = ["box_tracker_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:box_tracker_proto",
+    ],
+)
+
+proto_library(
+    name = "video_pre_stream_calculator_proto",
+    srcs = ["video_pre_stream_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "motion_analysis_calculator_cc_proto",
+    srcs = ["motion_analysis_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:motion_analysis_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":motion_analysis_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "flow_packager_calculator_cc_proto",
+    srcs = ["flow_packager_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:flow_packager_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":flow_packager_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "box_tracker_calculator_cc_proto",
+    srcs = ["box_tracker_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":box_tracker_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "video_pre_stream_calculator_cc_proto",
+    srcs = ["video_pre_stream_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":video_pre_stream_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "flow_to_image_calculator_cc_proto",
     srcs = ["flow_to_image_calculator.proto"],
@@ -131,6 +211,107 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "motion_analysis_calculator",
+    srcs = ["motion_analysis_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":motion_analysis_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking:camera_motion",
+        "//mediapipe/util/tracking:camera_motion_cc_proto",
+        "//mediapipe/util/tracking:frame_selection_cc_proto",
+        "//mediapipe/util/tracking:motion_analysis",
+        "//mediapipe/util/tracking:motion_estimation",
+        "//mediapipe/util/tracking:motion_models",
+        "//mediapipe/util/tracking:region_flow_cc_proto",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "flow_packager_calculator",
+    srcs = ["flow_packager_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":flow_packager_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/util/tracking:camera_motion_cc_proto",
+        "//mediapipe/util/tracking:flow_packager",
+        "//mediapipe/util/tracking:region_flow_cc_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "box_tracker_calculator",
+    srcs = ["box_tracker_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":box_tracker_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",  # fixdeps: keep -- required for exobazel build.
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:options_util",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "tracked_detection_manager_calculator",
+    srcs = ["tracked_detection_manager_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:tracked_detection",
+        "//mediapipe/util/tracking:tracked_detection_manager",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+        "@com_google_absl//absl/container:node_hash_map",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "video_pre_stream_calculator",
+    srcs = ["video_pre_stream_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":video_pre_stream_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:video_stream_header",
+    ],
+    alwayslink = 1,
+)
+
 filegroup(
     name = "test_videos",
     srcs = [
@@ -201,3 +382,78 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
     ],
 )
+
+MEDIAPIPE_DEPS = [
+    "//mediapipe/calculators/video:box_tracker_calculator",
+    "//mediapipe/calculators/video:flow_packager_calculator",
+    "//mediapipe/calculators/video:motion_analysis_calculator",
+    "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
+    "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+]
+
+mediapipe_binary_graph(
+    name = "parallel_tracker_binarypb",
+    graph = "testdata/parallel_tracker_graph.pbtxt",
+    output_name = "testdata/parallel_tracker.binarypb",
+    visibility = ["//visibility:public"],
+    deps = MEDIAPIPE_DEPS,
+)
+
+mediapipe_binary_graph(
+    name = "tracker_binarypb",
+    graph = "testdata/tracker_graph.pbtxt",
+    output_name = "testdata/tracker.binarypb",
+    visibility = ["//visibility:public"],
+    deps = MEDIAPIPE_DEPS,
+)
+
+cc_test(
+    name = "tracking_graph_test",
+    size = "small",
+    srcs = ["tracking_graph_test.cc"],
+    copts = ["-DPARALLEL_INVOKER_ACTIVE"] + select({
+        "//mediapipe:apple": [],
+        "//mediapipe:android": [],
+        "//conditions:default": [],
+    }),
+    data = [
+        ":testdata/lenna.png",
+        ":testdata/parallel_tracker.binarypb",
+        ":testdata/tracker.binarypb",
+    ],
+    deps = [
+        ":box_tracker_calculator",
+        ":box_tracker_calculator_cc_proto",
+        ":flow_packager_calculator",
+        ":motion_analysis_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
+        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:tracking_cc_proto",
+    ],
+)
+
+cc_test(
+    name = "video_pre_stream_calculator_test",
+    srcs = ["video_pre_stream_calculator_test.cc"],
+    deps = [
+        ":video_pre_stream_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/calculators/video/box_tracker_calculator.proto

@@ -0,0 +1,55 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/box_tracker.proto";
+
+message BoxTrackerCalculatorOptions {
+  extend CalculatorOptions {
+    optional BoxTrackerCalculatorOptions ext = 268767860;
+  }
+
+  optional BoxTrackerOptions tracker_options = 1;
+
+  // Initial position to be tracked. Can also be supplied as side packet or
+  // as input stream.
+  optional TimedBoxProtoList initial_position = 2;
+
+  // If set and VIZ stream is present, renders tracking data into the
+  // visualization.
+  optional bool visualize_tracking_data = 3 [default = false];
+
+  // If set and VIZ stream is present, renders the box state
+  // into the visualization.
+  optional bool visualize_state = 4 [default = false];
+
+  // If set and VIZ stream is present, renders the internal box state
+  // into the visualization.
+  optional bool visualize_internal_state = 5 [default = false];
+
+  // Size of the track data cache during streaming mode. This allows to buffer
+  // track_data's for fast forward tracking, i.e. any TimedBox received
+  // via input stream START_POS can be tracked towards the current track head
+  // (i.e. last received TrackingData). Measured in number of frames.
+  optional int32 streaming_track_data_cache_size = 6 [default = 0];
+
+  // Add a transition period of N frames to smooth the jump from original
+  // tracking to reset start pos with motion compensation. The transition will
+  // be a linear decay of original tracking result. 0 means no transition.
+  optional int32 start_pos_transition_frames = 7 [default = 0];
+}

mediapipe/calculators/video/flow_packager_calculator.cc

@@ -0,0 +1,281 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <fstream>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/calculators/video/flow_packager_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/util/tracking/camera_motion.pb.h"
+#include "mediapipe/util/tracking/flow_packager.h"
+#include "mediapipe/util/tracking/region_flow.pb.h"
+
+namespace mediapipe {
+
+using mediapipe::CameraMotion;
+using mediapipe::FlowPackager;
+using mediapipe::RegionFlowFeatureList;
+using mediapipe::TrackingData;
+using mediapipe::TrackingDataChunk;
+
+// A calculator that packages input CameraMotion and RegionFlowFeatureList
+// into a TrackingData and optionally writes TrackingDataChunks to file.
+//
+// Input stream:
+//   FLOW:       Input region flow (proto RegionFlowFeatureList).
+//   CAMERA:     Input camera stream (proto CameraMotion, optional).
+//
+// Input side packets:
+//   CACHE_DIR:  Optional caching directory tracking files are written to.
+//
+// Output streams.
+//   TRACKING:       Output tracking data (proto TrackingData, per frame
+//                   optional).
+//   TRACKING_CHUNK: Output tracking chunks (proto TrackingDataChunk,
+//                   per chunk, optional), output at the first timestamp
+//                   of each chunk.
+//   COMPLETE:       Optional output packet sent on PreStream to
+//                   to signal downstream calculators that all data has been
+//                   processed and calculator is closed. Can be used to indicate
+//                   that all data as been written to CACHE_DIR.
+class FlowPackagerCalculator : public CalculatorBase {
+ public:
+  ~FlowPackagerCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+  // Writes passed chunk to disk.
+  void WriteChunk(const TrackingDataChunk& chunk) const;
+
+  // Initializes next chunk for tracking beginning from last frame of
+  // current chunk (Chunking is design with one frame overlap).
+  void PrepareCurrentForNextChunk(TrackingDataChunk* chunk);
+
+ private:
+  FlowPackagerCalculatorOptions options_;
+
+  // Caching options.
+  bool use_caching_ = false;
+  bool build_chunk_ = false;
+  std::string cache_dir_;
+  int chunk_idx_ = -1;
+  TrackingDataChunk tracking_chunk_;
+
+  int frame_idx_ = 0;
+
+  Timestamp prev_timestamp_;
+  std::unique_ptr<FlowPackager> flow_packager_;
+};
+
+REGISTER_CALCULATOR(FlowPackagerCalculator);
+
+::mediapipe::Status FlowPackagerCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (!cc->Inputs().HasTag("FLOW")) {
+    return tool::StatusFail("No input flow was specified.");
+  }
+
+  cc->Inputs().Tag("FLOW").Set<RegionFlowFeatureList>();
+
+  if (cc->Inputs().HasTag("CAMERA")) {
+    cc->Inputs().Tag("CAMERA").Set<CameraMotion>();
+  }
+  if (cc->Outputs().HasTag("TRACKING")) {
+    cc->Outputs().Tag("TRACKING").Set<TrackingData>();
+  }
+  if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+    cc->Outputs().Tag("TRACKING_CHUNK").Set<TrackingDataChunk>();
+  }
+  if (cc->Outputs().HasTag("COMPLETE")) {
+    cc->Outputs().Tag("COMPLETE").Set<bool>();
+  }
+
+  if (cc->InputSidePackets().HasTag("CACHE_DIR")) {
+    cc->InputSidePackets().Tag("CACHE_DIR").Set<std::string>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<FlowPackagerCalculatorOptions>();
+
+  flow_packager_.reset(new FlowPackager(options_.flow_packager_options()));
+
+  use_caching_ = cc->InputSidePackets().HasTag("CACHE_DIR");
+  build_chunk_ = use_caching_ || cc->Outputs().HasTag("TRACKING_CHUNK");
+  if (use_caching_) {
+    cache_dir_ = cc->InputSidePackets().Tag("CACHE_DIR").Get<std::string>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Process(CalculatorContext* cc) {
+  InputStream* flow_stream = &(cc->Inputs().Tag("FLOW"));
+  const RegionFlowFeatureList& flow = flow_stream->Get<RegionFlowFeatureList>();
+
+  const Timestamp timestamp = flow_stream->Value().Timestamp();
+
+  const CameraMotion* camera_motion = nullptr;
+  if (cc->Inputs().HasTag("CAMERA")) {
+    InputStream* camera_stream = &(cc->Inputs().Tag("CAMERA"));
+    camera_motion = &camera_stream->Get<CameraMotion>();
+  }
+
+  std::unique_ptr<TrackingData> tracking_data(new TrackingData());
+
+  flow_packager_->PackFlow(flow, camera_motion, tracking_data.get());
+
+  if (build_chunk_) {
+    if (chunk_idx_ < 0) {  // Lazy init, determine first start.
+      chunk_idx_ =
+          timestamp.Value() / 1000 / options_.caching_chunk_size_msec();
+      tracking_chunk_.set_first_chunk(true);
+    }
+    CHECK_GE(chunk_idx_, 0);
+
+    TrackingDataChunk::Item* item = tracking_chunk_.add_item();
+    item->set_frame_idx(frame_idx_);
+    item->set_timestamp_usec(timestamp.Value());
+    if (frame_idx_ > 0) {
+      item->set_prev_timestamp_usec(prev_timestamp_.Value());
+    }
+    if (cc->Outputs().HasTag("TRACKING")) {
+      // Need to copy as output is requested.
+      *item->mutable_tracking_data() = *tracking_data;
+    } else {
+      item->mutable_tracking_data()->Swap(tracking_data.get());
+    }
+
+    const int next_chunk_msec =
+        options_.caching_chunk_size_msec() * (chunk_idx_ + 1);
+
+    if (timestamp.Value() / 1000 >= next_chunk_msec) {
+      if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+        cc->Outputs()
+            .Tag("TRACKING_CHUNK")
+            .Add(new TrackingDataChunk(tracking_chunk_),
+                 Timestamp(tracking_chunk_.item(0).timestamp_usec()));
+      }
+      if (use_caching_) {
+        WriteChunk(tracking_chunk_);
+      }
+      PrepareCurrentForNextChunk(&tracking_chunk_);
+    }
+  }
+
+  if (cc->Outputs().HasTag("TRACKING")) {
+    cc->Outputs()
+        .Tag("TRACKING")
+        .Add(tracking_data.release(), flow_stream->Value().Timestamp());
+  }
+
+  prev_timestamp_ = timestamp;
+  ++frame_idx_;
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FlowPackagerCalculator::Close(CalculatorContext* cc) {
+  if (frame_idx_ > 0) {
+    tracking_chunk_.set_last_chunk(true);
+    if (cc->Outputs().HasTag("TRACKING_CHUNK")) {
+      cc->Outputs()
+          .Tag("TRACKING_CHUNK")
+          .Add(new TrackingDataChunk(tracking_chunk_),
+               Timestamp(tracking_chunk_.item(0).timestamp_usec()));
+    }
+
+    if (use_caching_) {
+      WriteChunk(tracking_chunk_);
+    }
+  }
+
+  if (cc->Outputs().HasTag("COMPLETE")) {
+    cc->Outputs().Tag("COMPLETE").Add(new bool(true), Timestamp::PreStream());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const {
+  if (chunk.item_size() == 0) {
+    LOG(ERROR) << "Write chunk called with empty tracking data."
+               << "This can only occur if the spacing between frames "
+               << "is larger than the requested chunk size. Try increasing "
+               << "the chunk size";
+    return;
+  }
+
+  auto format_runtime =
+      absl::ParsedFormat<'d'>::New(options_.cache_file_format());
+
+  std::string chunk_file;
+  if (format_runtime) {
+    chunk_file =
+        cache_dir_ + "/" + absl::StrFormat(*format_runtime, chunk_idx_);
+  } else {
+    LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d.";
+    chunk_file = cache_dir_ + "/" + absl::StrFormat("chunk_%04d", chunk_idx_);
+  }
+
+  std::string data;
+  chunk.SerializeToString(&data);
+
+  const char* temp_filename = tempnam(cache_dir_.c_str(), nullptr);
+  std::ofstream out_file(temp_filename);
+  if (!out_file) {
+    LOG(ERROR) << "Could not open " << temp_filename;
+  } else {
+    out_file.write(data.data(), data.size());
+  }
+
+  if (rename(temp_filename, chunk_file.c_str()) != 0) {
+    LOG(ERROR) << "Failed to rename to " << chunk_file;
+  }
+
+  LOG(INFO) << "Wrote chunk : " << chunk_file;
+}
+
+void FlowPackagerCalculator::PrepareCurrentForNextChunk(
+    TrackingDataChunk* chunk) {
+  CHECK(chunk);
+  if (chunk->item_size() == 0) {
+    LOG(ERROR) << "Called with empty chunk. Unexpected.";
+    return;
+  }
+
+  chunk->set_first_chunk(false);
+
+  // Buffer last item for next chunk.
+  TrackingDataChunk::Item last_item;
+  last_item.Swap(chunk->mutable_item(chunk->item_size() - 1));
+
+  chunk->Clear();
+  chunk->add_item()->Swap(&last_item);
+
+  ++chunk_idx_;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/flow_packager_calculator.proto

@@ -0,0 +1,36 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/flow_packager.proto";
+
+message FlowPackagerCalculatorOptions {
+  extend CalculatorOptions {
+    optional FlowPackagerCalculatorOptions ext = 271236147;
+  }
+
+  optional mediapipe.FlowPackagerOptions flow_packager_options = 1;
+
+  // Chunk size for caching files that are written to the externally specified
+  // caching directory. Specified in msec.
+  // Note that each chunk always contains at its end the first frame of the
+  // next chunk (to enable forward tracking across chunk boundaries).
+  optional int32 caching_chunk_size_msec = 2 [default = 2500];
+
+  optional string cache_file_format = 3 [default = "chunk_%04d"];
+}

mediapipe/calculators/video/motion_analysis_calculator.cc

@@ -0,0 +1,988 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <fstream>
+#include <memory>
+
+#include "absl/strings/numbers.h"
+#include "absl/strings/str_split.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/calculators/video/motion_analysis_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/camera_motion.h"
+#include "mediapipe/util/tracking/camera_motion.pb.h"
+#include "mediapipe/util/tracking/frame_selection.pb.h"
+#include "mediapipe/util/tracking/motion_analysis.h"
+#include "mediapipe/util/tracking/motion_estimation.h"
+#include "mediapipe/util/tracking/motion_models.h"
+#include "mediapipe/util/tracking/region_flow.pb.h"
+
+namespace mediapipe {
+
+using mediapipe::AffineAdapter;
+using mediapipe::CameraMotion;
+using mediapipe::FrameSelectionResult;
+using mediapipe::Homography;
+using mediapipe::HomographyAdapter;
+using mediapipe::LinearSimilarityModel;
+using mediapipe::MixtureHomography;
+using mediapipe::MixtureRowWeights;
+using mediapipe::MotionAnalysis;
+using mediapipe::ProjectViaFit;
+using mediapipe::RegionFlowComputationOptions;
+using mediapipe::RegionFlowFeatureList;
+using mediapipe::SalientPointFrame;
+using mediapipe::TranslationModel;
+
+const char kOptionsTag[] = "OPTIONS";
+
+// A calculator that performs motion analysis on an incoming video stream.
+//
+// Input streams:  (at least one of them is required).
+//   VIDEO:     The input video stream (ImageFrame, sRGB, sRGBA or GRAY8).
+//   SELECTION: Optional input stream to perform analysis only on selected
+//              frames. If present needs to contain camera motion
+//              and features.
+//
+// Input side packets:
+//   CSV_FILE:  Read motion models as homographies from CSV file. Expected
+//              to be defined in the frame domain (un-normalized).
+//              Should store 9 floats per row.
+//              Specify number of homographies per frames via option
+//              meta_models_per_frame. For values > 1, MixtureHomographies
+//              are created, for value == 1, a single Homography is used.
+//   DOWNSAMPLE: Optionally specify downsampling factor via input side packet
+//               overriding value in the graph settings.
+// Output streams (all are optional).
+//   FLOW:      Sparse feature tracks in form of proto RegionFlowFeatureList.
+//   CAMERA:    Camera motion as proto CameraMotion describing the per frame-
+//              pair motion. Has VideoHeader from input video.
+//   SALIENCY:  Foreground saliency (objects moving different from the
+//              background) as proto SalientPointFrame.
+//   VIZ:       Visualization stream as ImageFrame, sRGB, visualizing
+//              features and saliency (set via
+//              analysis_options().visualization_options())
+//   DENSE_FG:  Dense foreground stream, describing per-pixel foreground-
+//              ness as confidence between 0 (background) and 255
+//              (foreground). Output is ImageFrame (GRAY8).
+//   VIDEO_OUT: Optional output stream when SELECTION is used. Output is input
+//              VIDEO at the selected frames. Required VIDEO to be present.
+//   GRAY_VIDEO_OUT: Optional output stream for downsampled, grayscale video.
+//                   Requires VIDEO to be present and SELECTION to not be used.
+class MotionAnalysisCalculator : public CalculatorBase {
+  // TODO: Activate once leakr approval is ready.
+  // typedef com::google::android::libraries::micro::proto::Data HomographyData;
+
+ public:
+  ~MotionAnalysisCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  // Outputs results to Outputs() if MotionAnalysis buffered sufficient results.
+  // Otherwise no-op. Set flush to true to force output of all buffered data.
+  void OutputMotionAnalyzedFrames(bool flush, CalculatorContext* cc);
+
+  // Lazy init function to be called on Process.
+  ::mediapipe::Status InitOnProcess(InputStream* video_stream,
+                                    InputStream* selection_stream);
+
+  // Parses CSV file contents to homographies.
+  bool ParseModelCSV(const std::string& contents,
+                     std::deque<Homography>* homographies);
+
+  // Turns list of 9-tuple floating values into set of homographies.
+  bool HomographiesFromValues(const std::vector<float>& homog_values,
+                              std::deque<Homography>* homographies);
+
+  // Appends CameraMotions and features from homographies.
+  // Set append_identity to true to add an identity transform to the beginning
+  // of the each list *in addition* to the motions derived from homographies.
+  void AppendCameraMotionsFromHomographies(
+      const std::deque<Homography>& homographies, bool append_identity,
+      std::deque<CameraMotion>* camera_motions,
+      std::deque<RegionFlowFeatureList>* features);
+
+  // Helper function to subtract current metadata motion from features. Used
+  // for hybrid estimation case.
+  void SubtractMetaMotion(const CameraMotion& meta_motion,
+                          RegionFlowFeatureList* features);
+
+  // Inverse of above function to add back meta motion and replace
+  // feature location with originals after estimation.
+  void AddMetaMotion(const CameraMotion& meta_motion,
+                     const RegionFlowFeatureList& meta_features,
+                     RegionFlowFeatureList* features, CameraMotion* motion);
+
+  MotionAnalysisCalculatorOptions options_;
+  int frame_width_ = -1;
+  int frame_height_ = -1;
+  int frame_idx_ = 0;
+
+  // Buffers incoming video frame packets (if visualization output is requested)
+  std::vector<Packet> packet_buffer_;
+
+  // Buffers incoming timestamps until MotionAnalysis is ready to output via
+  // above OutputMotionAnalyzedFrames.
+  std::vector<Timestamp> timestamp_buffer_;
+
+  // Input indicators for each stream.
+  bool selection_input_ = false;
+  bool video_input_ = false;
+
+  // Output indicators for each stream.
+  bool region_flow_feature_output_ = false;
+  bool camera_motion_output_ = false;
+  bool saliency_output_ = false;
+  bool visualize_output_ = false;
+  bool dense_foreground_output_ = false;
+  bool video_output_ = false;
+  bool grayscale_output_ = false;
+  bool csv_file_input_ = false;
+
+  // Inidicates if saliency should be computed.
+  bool with_saliency_ = false;
+
+  // Set if hybrid meta analysis - see proto for details.
+  bool hybrid_meta_analysis_ = false;
+
+  // Concatenated motions for each selected frame. Used in case
+  // hybrid estimation is requested to fallback to valid models.
+  std::deque<CameraMotion> selected_motions_;
+
+  // Normalized homographies from CSV file or metadata.
+  std::deque<Homography> meta_homographies_;
+  std::deque<CameraMotion> meta_motions_;
+  std::deque<RegionFlowFeatureList> meta_features_;
+
+  // Offset into above meta_motions_ and features_ when using
+  // hybrid meta analysis.
+  int hybrid_meta_offset_ = 0;
+
+  std::unique_ptr<MotionAnalysis> motion_analysis_;
+
+  std::unique_ptr<MixtureRowWeights> row_weights_;
+};
+
+REGISTER_CALCULATOR(MotionAnalysisCalculator);
+
+::mediapipe::Status MotionAnalysisCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("VIDEO")) {
+    cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  }
+
+  // Optional input stream from frame selection calculator.
+  if (cc->Inputs().HasTag("SELECTION")) {
+    cc->Inputs().Tag("SELECTION").Set<FrameSelectionResult>();
+  }
+
+  RET_CHECK(cc->Inputs().HasTag("VIDEO") || cc->Inputs().HasTag("SELECTION"))
+      << "Either VIDEO, SELECTION must be specified.";
+
+  if (cc->Outputs().HasTag("FLOW")) {
+    cc->Outputs().Tag("FLOW").Set<RegionFlowFeatureList>();
+  }
+
+  if (cc->Outputs().HasTag("CAMERA")) {
+    cc->Outputs().Tag("CAMERA").Set<CameraMotion>();
+  }
+
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    cc->Outputs().Tag("SALIENCY").Set<SalientPointFrame>();
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    cc->Outputs().Tag("VIZ").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("DENSE_FG")) {
+    cc->Outputs().Tag("DENSE_FG").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("VIDEO_OUT")) {
+    cc->Outputs().Tag("VIDEO_OUT").Set<ImageFrame>();
+  }
+
+  if (cc->Outputs().HasTag("GRAY_VIDEO_OUT")) {
+    // We only output grayscale video if we're actually performing full region-
+    // flow analysis on the video.
+    RET_CHECK(cc->Inputs().HasTag("VIDEO") &&
+              !cc->Inputs().HasTag("SELECTION"));
+    cc->Outputs().Tag("GRAY_VIDEO_OUT").Set<ImageFrame>();
+  }
+
+  if (cc->InputSidePackets().HasTag("CSV_FILE")) {
+    cc->InputSidePackets().Tag("CSV_FILE").Set<std::string>();
+  }
+  if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) {
+    cc->InputSidePackets().Tag("DOWNSAMPLE").Set<float>();
+  }
+
+  if (cc->InputSidePackets().HasTag(kOptionsTag)) {
+    cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Open(CalculatorContext* cc) {
+  options_ =
+      tool::RetrieveOptions(cc->Options<MotionAnalysisCalculatorOptions>(),
+                            cc->InputSidePackets(), kOptionsTag);
+
+  video_input_ = cc->Inputs().HasTag("VIDEO");
+  selection_input_ = cc->Inputs().HasTag("SELECTION");
+  region_flow_feature_output_ = cc->Outputs().HasTag("FLOW");
+  camera_motion_output_ = cc->Outputs().HasTag("CAMERA");
+  saliency_output_ = cc->Outputs().HasTag("SALIENCY");
+  visualize_output_ = cc->Outputs().HasTag("VIZ");
+  dense_foreground_output_ = cc->Outputs().HasTag("DENSE_FG");
+  video_output_ = cc->Outputs().HasTag("VIDEO_OUT");
+  grayscale_output_ = cc->Outputs().HasTag("GRAY_VIDEO_OUT");
+  csv_file_input_ = cc->InputSidePackets().HasTag("CSV_FILE");
+  hybrid_meta_analysis_ = options_.meta_analysis() ==
+                          MotionAnalysisCalculatorOptions::META_ANALYSIS_HYBRID;
+
+  if (video_output_) {
+    RET_CHECK(selection_input_) << "VIDEO_OUT requires SELECTION input";
+  }
+
+  if (selection_input_) {
+    switch (options_.selection_analysis()) {
+      case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION:
+        RET_CHECK(!visualize_output_)
+            << "Visualization not supported for NO_ANALYSIS_USE_SELECTION";
+        RET_CHECK(!dense_foreground_output_)
+            << "Dense foreground not supported for NO_ANALYSIS_USE_SELECTION";
+        RET_CHECK(!saliency_output_)
+            << "Saliency output not supported for NO_ANALYSIS_USE_SELECTION";
+        break;
+
+      case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE:
+      case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED:
+        RET_CHECK(video_input_) << "Need video input for feature tracking.";
+        break;
+
+      case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES:
+        // Nothing to add here.
+        break;
+    }
+  }
+
+  if (visualize_output_ || dense_foreground_output_ || video_output_) {
+    RET_CHECK(video_input_) << "Video input required.";
+  }
+
+  if (csv_file_input_) {
+    RET_CHECK(!selection_input_)
+        << "Can not use selection input with csv input.";
+    if (!hybrid_meta_analysis_) {
+      RET_CHECK(!saliency_output_ && !visualize_output_ &&
+                !dense_foreground_output_ && !grayscale_output_)
+          << "CSV file and meta input only supports flow and camera motion "
+          << "output when using metadata only.";
+    }
+  }
+
+  if (csv_file_input_) {
+    // Read from file and parse.
+    const std::string filename =
+        cc->InputSidePackets().Tag("CSV_FILE").Get<std::string>();
+
+    std::string file_contents;
+    std::ifstream input_file(filename, std::ios::in);
+    input_file.seekg(0, std::ios::end);
+    const int file_length = input_file.tellg();
+    file_contents.resize(file_length);
+    input_file.seekg(0, std::ios::beg);
+    input_file.read(&file_contents[0], file_length);
+    input_file.close();
+
+    RET_CHECK(ParseModelCSV(file_contents, &meta_homographies_))
+        << "Could not parse CSV file";
+  }
+
+  // Get video header from video or selection input if present.
+  const VideoHeader* video_header = nullptr;
+  if (video_input_ && !cc->Inputs().Tag("VIDEO").Header().IsEmpty()) {
+    video_header = &(cc->Inputs().Tag("VIDEO").Header().Get<VideoHeader>());
+  } else if (selection_input_ &&
+             !cc->Inputs().Tag("SELECTION").Header().IsEmpty()) {
+    video_header = &(cc->Inputs().Tag("SELECTION").Header().Get<VideoHeader>());
+  } else {
+    LOG(WARNING) << "No input video header found. Downstream calculators "
+                    "expecting video headers are likely to fail.";
+  }
+
+  with_saliency_ = options_.analysis_options().compute_motion_saliency();
+  // Force computation of saliency if requested as output.
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    with_saliency_ = true;
+    if (!options_.analysis_options().compute_motion_saliency()) {
+      LOG(WARNING) << "Enable saliency computation. Set "
+                   << "compute_motion_saliency to true to silence this "
+                   << "warning.";
+      options_.mutable_analysis_options()->set_compute_motion_saliency(true);
+    }
+  }
+
+  if (options_.bypass_mode()) {
+    cc->SetOffset(TimestampDiff(0));
+  }
+
+  if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) {
+    options_.mutable_analysis_options()
+        ->mutable_flow_options()
+        ->set_downsample_factor(
+            cc->InputSidePackets().Tag("DOWNSAMPLE").Get<float>());
+  }
+
+  // If no video header is provided, just return and initialize on the first
+  // Process() call.
+  if (video_header == nullptr) {
+    return ::mediapipe::OkStatus();
+  }
+
+  ////////////// EARLY RETURN; ONLY HEADER OUTPUT SHOULD GO HERE ///////////////
+
+  if (visualize_output_) {
+    cc->Outputs().Tag("VIZ").SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (video_output_) {
+    cc->Outputs()
+        .Tag("VIDEO_OUT")
+        .SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (cc->Outputs().HasTag("DENSE_FG")) {
+    std::unique_ptr<VideoHeader> foreground_header(
+        new VideoHeader(*video_header));
+    foreground_header->format = ImageFormat::GRAY8;
+    cc->Outputs().Tag("DENSE_FG").SetHeader(Adopt(foreground_header.release()));
+  }
+
+  if (cc->Outputs().HasTag("CAMERA")) {
+    cc->Outputs().Tag("CAMERA").SetHeader(
+        Adopt(new VideoHeader(*video_header)));
+  }
+
+  if (cc->Outputs().HasTag("SALIENCY")) {
+    cc->Outputs()
+        .Tag("SALIENCY")
+        .SetHeader(Adopt(new VideoHeader(*video_header)));
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) {
+  if (options_.bypass_mode()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  InputStream* video_stream =
+      video_input_ ? &(cc->Inputs().Tag("VIDEO")) : nullptr;
+  InputStream* selection_stream =
+      selection_input_ ? &(cc->Inputs().Tag("SELECTION")) : nullptr;
+
+  // Checked on Open.
+  CHECK(video_stream || selection_stream);
+
+  // Lazy init.
+  if (frame_width_ < 0 || frame_height_ < 0) {
+    MP_RETURN_IF_ERROR(InitOnProcess(video_stream, selection_stream));
+  }
+
+  const Timestamp timestamp = cc->InputTimestamp();
+  if ((csv_file_input_) && !hybrid_meta_analysis_) {
+    if (camera_motion_output_) {
+      RET_CHECK(!meta_motions_.empty()) << "Insufficient metadata.";
+
+      CameraMotion output_motion = meta_motions_.front();
+      meta_motions_.pop_front();
+      output_motion.set_timestamp_usec(timestamp.Value());
+      cc->Outputs().Tag("CAMERA").Add(new CameraMotion(output_motion),
+                                      timestamp);
+    }
+
+    if (region_flow_feature_output_) {
+      RET_CHECK(!meta_features_.empty()) << "Insufficient frames in CSV file";
+      RegionFlowFeatureList output_features = meta_features_.front();
+      meta_features_.pop_front();
+
+      output_features.set_timestamp_usec(timestamp.Value());
+      cc->Outputs().Tag("FLOW").Add(new RegionFlowFeatureList(output_features),
+                                    timestamp);
+    }
+
+    ++frame_idx_;
+    return ::mediapipe::OkStatus();
+  }
+
+  if (motion_analysis_ == nullptr) {
+    // We do not need MotionAnalysis when using just metadata.
+    motion_analysis_.reset(new MotionAnalysis(options_.analysis_options(),
+                                              frame_width_, frame_height_));
+  }
+
+  std::unique_ptr<FrameSelectionResult> frame_selection_result;
+  // Always use frame if selection is not activated.
+  bool use_frame = !selection_input_;
+  if (selection_input_) {
+    CHECK(selection_stream);
+
+    // Fill in timestamps we process.
+    if (!selection_stream->Value().IsEmpty()) {
+      ASSIGN_OR_RETURN(
+          frame_selection_result,
+          selection_stream->Value().ConsumeOrCopy<FrameSelectionResult>());
+      use_frame = true;
+
+      // Make sure both features and camera motion are present.
+      RET_CHECK(frame_selection_result->has_camera_motion() &&
+                frame_selection_result->has_features())
+          << "Frame selection input error at: " << timestamp
+          << " both camera motion and features need to be "
+             "present in FrameSelectionResult. "
+          << frame_selection_result->has_camera_motion() << " , "
+          << frame_selection_result->has_features();
+    }
+  }
+
+  if (selection_input_ && use_frame &&
+      options_.selection_analysis() ==
+          MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION) {
+    // Output concatenated results, nothing to compute here.
+    if (camera_motion_output_) {
+      cc->Outputs().Tag("CAMERA").Add(
+          frame_selection_result->release_camera_motion(), timestamp);
+    }
+    if (region_flow_feature_output_) {
+      cc->Outputs().Tag("FLOW").Add(frame_selection_result->release_features(),
+                                    timestamp);
+    }
+
+    if (video_output_) {
+      cc->Outputs().Tag("VIDEO_OUT").AddPacket(video_stream->Value());
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  if (use_frame) {
+    if (!selection_input_) {
+      const cv::Mat input_view =
+          formats::MatView(&video_stream->Get<ImageFrame>());
+      if (hybrid_meta_analysis_) {
+        // Seed with meta homography.
+        RET_CHECK(hybrid_meta_offset_ < meta_motions_.size())
+            << "Not enough metadata received for hybrid meta analysis";
+        Homography initial_transform =
+            meta_motions_[hybrid_meta_offset_].homography();
+        std::function<void(RegionFlowFeatureList*)> subtract_helper = std::bind(
+            &MotionAnalysisCalculator::SubtractMetaMotion, this,
+            meta_motions_[hybrid_meta_offset_], std::placeholders::_1);
+
+        // Keep original features before modification around.
+        motion_analysis_->AddFrameGeneric(
+            input_view, timestamp.Value(), initial_transform, nullptr, nullptr,
+            &subtract_helper, &meta_features_[hybrid_meta_offset_]);
+        ++hybrid_meta_offset_;
+      } else {
+        motion_analysis_->AddFrame(input_view, timestamp.Value());
+      }
+    } else {
+      selected_motions_.push_back(frame_selection_result->camera_motion());
+      switch (options_.selection_analysis()) {
+        case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION:
+          return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
+                 << "Should not reach this point!";
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES:
+          motion_analysis_->AddFeatures(frame_selection_result->features());
+          break;
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE: {
+          const cv::Mat input_view =
+              formats::MatView(&video_stream->Get<ImageFrame>());
+          motion_analysis_->AddFrame(input_view, timestamp.Value());
+          break;
+        }
+
+        case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED: {
+          Homography homography;
+          CameraMotionToHomography(frame_selection_result->camera_motion(),
+                                   &homography);
+          const cv::Mat input_view =
+              formats::MatView(&video_stream->Get<ImageFrame>());
+          motion_analysis_->AddFrameGeneric(input_view, timestamp.Value(),
+                                            homography, &homography);
+          break;
+        }
+      }
+    }
+
+    timestamp_buffer_.push_back(timestamp);
+    ++frame_idx_;
+
+    VLOG_EVERY_N(0, 100) << "Analyzed frame " << frame_idx_;
+
+    // Buffer input frames only if visualization is requested.
+    if (visualize_output_ || video_output_) {
+      packet_buffer_.push_back(video_stream->Value());
+    }
+
+    // If requested, output grayscale thumbnails
+    if (grayscale_output_) {
+      cv::Mat grayscale_mat = motion_analysis_->GetGrayscaleFrameFromResults();
+      std::unique_ptr<ImageFrame> grayscale_image(new ImageFrame(
+          ImageFormat::GRAY8, grayscale_mat.cols, grayscale_mat.rows));
+      cv::Mat image_frame_mat = formats::MatView(grayscale_image.get());
+      grayscale_mat.copyTo(image_frame_mat);
+
+      cc->Outputs()
+          .Tag("GRAY_VIDEO_OUT")
+          .Add(grayscale_image.release(), timestamp);
+    }
+
+    // Output other results, if we have any yet.
+    OutputMotionAnalyzedFrames(false, cc);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MotionAnalysisCalculator::Close(CalculatorContext* cc) {
+  // Guard against empty videos.
+  if (motion_analysis_) {
+    OutputMotionAnalyzedFrames(true, cc);
+  }
+  if (csv_file_input_) {
+    if (!meta_motions_.empty()) {
+      LOG(ERROR) << "More motions than frames. Unexpected! Remainder: "
+                 << meta_motions_.size();
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+void MotionAnalysisCalculator::OutputMotionAnalyzedFrames(
+    bool flush, CalculatorContext* cc) {
+  std::vector<std::unique_ptr<RegionFlowFeatureList>> features;
+  std::vector<std::unique_ptr<CameraMotion>> camera_motions;
+  std::vector<std::unique_ptr<SalientPointFrame>> saliency;
+
+  const int buffer_size = timestamp_buffer_.size();
+  const int num_results = motion_analysis_->GetResults(
+      flush, &features, &camera_motions, with_saliency_ ? &saliency : nullptr);
+
+  CHECK_LE(num_results, buffer_size);
+
+  if (num_results == 0) {
+    return;
+  }
+
+  for (int k = 0; k < num_results; ++k) {
+    // Region flow features and camera motion for this frame.
+    auto& feature_list = features[k];
+    auto& camera_motion = camera_motions[k];
+    const Timestamp timestamp = timestamp_buffer_[k];
+
+    if (selection_input_ && options_.hybrid_selection_camera()) {
+      if (camera_motion->type() > selected_motions_.front().type()) {
+        // Composited type is more stable.
+        camera_motion->Swap(&selected_motions_.front());
+      }
+      selected_motions_.pop_front();
+    }
+
+    if (hybrid_meta_analysis_) {
+      AddMetaMotion(meta_motions_.front(), meta_features_.front(),
+                    feature_list.get(), camera_motion.get());
+      meta_motions_.pop_front();
+      meta_features_.pop_front();
+    }
+
+    // Video frame for visualization.
+    std::unique_ptr<ImageFrame> visualization_frame;
+    cv::Mat visualization;
+    if (visualize_output_) {
+      // Initialize visualization frame with original frame.
+      visualization_frame.reset(new ImageFrame());
+      visualization_frame->CopyFrom(packet_buffer_[k].Get<ImageFrame>(), 16);
+      visualization = formats::MatView(visualization_frame.get());
+
+      motion_analysis_->RenderResults(
+          *feature_list, *camera_motion,
+          with_saliency_ ? saliency[k].get() : nullptr, &visualization);
+
+      cc->Outputs().Tag("VIZ").Add(visualization_frame.release(), timestamp);
+    }
+
+    // Output dense foreground mask.
+    if (dense_foreground_output_) {
+      std::unique_ptr<ImageFrame> foreground_frame(
+          new ImageFrame(ImageFormat::GRAY8, frame_width_, frame_height_));
+      cv::Mat foreground = formats::MatView(foreground_frame.get());
+      motion_analysis_->ComputeDenseForeground(*feature_list, *camera_motion,
+                                               &foreground);
+      cc->Outputs().Tag("DENSE_FG").Add(foreground_frame.release(), timestamp);
+    }
+
+    // Output flow features if requested.
+    if (region_flow_feature_output_) {
+      cc->Outputs().Tag("FLOW").Add(feature_list.release(), timestamp);
+    }
+
+    // Output camera motion.
+    if (camera_motion_output_) {
+      cc->Outputs().Tag("CAMERA").Add(camera_motion.release(), timestamp);
+    }
+
+    if (video_output_) {
+      cc->Outputs().Tag("VIDEO_OUT").AddPacket(packet_buffer_[k]);
+    }
+
+    // Output saliency.
+    if (saliency_output_) {
+      cc->Outputs().Tag("SALIENCY").Add(saliency[k].release(), timestamp);
+    }
+  }
+
+  if (hybrid_meta_analysis_) {
+    hybrid_meta_offset_ -= num_results;
+    CHECK_GE(hybrid_meta_offset_, 0);
+  }
+
+  timestamp_buffer_.erase(timestamp_buffer_.begin(),
+                          timestamp_buffer_.begin() + num_results);
+
+  if (visualize_output_ || video_output_) {
+    packet_buffer_.erase(packet_buffer_.begin(),
+                         packet_buffer_.begin() + num_results);
+  }
+}
+
+::mediapipe::Status MotionAnalysisCalculator::InitOnProcess(
+    InputStream* video_stream, InputStream* selection_stream) {
+  if (video_stream) {
+    frame_width_ = video_stream->Get<ImageFrame>().Width();
+    frame_height_ = video_stream->Get<ImageFrame>().Height();
+
+    // Ensure image options are set correctly.
+    auto* region_options =
+        options_.mutable_analysis_options()->mutable_flow_options();
+
+    // Use two possible formats to account for different channel orders.
+    RegionFlowComputationOptions::ImageFormat image_format;
+    RegionFlowComputationOptions::ImageFormat image_format2;
+    switch (video_stream->Get<ImageFrame>().Format()) {
+      case ImageFormat::GRAY8:
+        image_format = image_format2 =
+            RegionFlowComputationOptions::FORMAT_GRAYSCALE;
+        break;
+
+      case ImageFormat::SRGB:
+        image_format = RegionFlowComputationOptions::FORMAT_RGB;
+        image_format2 = RegionFlowComputationOptions::FORMAT_BGR;
+        break;
+
+      case ImageFormat::SRGBA:
+        image_format = RegionFlowComputationOptions::FORMAT_RGBA;
+        image_format2 = RegionFlowComputationOptions::FORMAT_BGRA;
+        break;
+
+      default:
+        RET_CHECK(false) << "Unsupported image format.";
+    }
+    if (region_options->image_format() != image_format &&
+        region_options->image_format() != image_format2) {
+      LOG(WARNING) << "Requested image format in RegionFlowComputation "
+                   << "does not match video stream format. Overriding.";
+      region_options->set_image_format(image_format);
+    }
+
+    // Account for downsampling mode INPUT_SIZE. In this case we are handed
+    // already downsampled frames but the resulting CameraMotion should
+    // be computed on higher resolution as specifed by the downsample scale.
+    if (region_options->downsample_mode() ==
+        RegionFlowComputationOptions::DOWNSAMPLE_TO_INPUT_SIZE) {
+      const float scale = region_options->downsample_factor();
+      frame_width_ = static_cast<int>(std::round(frame_width_ * scale));
+      frame_height_ = static_cast<int>(std::round(frame_height_ * scale));
+    }
+  } else if (selection_stream) {
+    const auto& camera_motion =
+        selection_stream->Get<FrameSelectionResult>().camera_motion();
+    frame_width_ = camera_motion.frame_width();
+    frame_height_ = camera_motion.frame_height();
+  } else {
+    LOG(FATAL) << "Either VIDEO or SELECTION stream need to be specified.";
+  }
+
+  // Filled by CSV file parsing.
+  if (!meta_homographies_.empty()) {
+    CHECK(csv_file_input_);
+    AppendCameraMotionsFromHomographies(meta_homographies_,
+                                        true,  // append identity.
+                                        &meta_motions_, &meta_features_);
+    meta_homographies_.clear();
+  }
+
+  // Filter weights before using for hybrid mode.
+  if (hybrid_meta_analysis_) {
+    auto* motion_options =
+        options_.mutable_analysis_options()->mutable_motion_options();
+    motion_options->set_filter_initialized_irls_weights(true);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+bool MotionAnalysisCalculator::ParseModelCSV(
+    const std::string& contents, std::deque<Homography>* homographies) {
+  std::vector<absl::string_view> values =
+      absl::StrSplit(contents, absl::ByAnyChar(",\n"));
+
+  // Trim off any empty lines.
+  while (values.back().empty()) {
+    values.pop_back();
+  }
+
+  // Convert to float.
+  std::vector<float> homog_values;
+  homog_values.reserve(values.size());
+
+  for (const auto& value : values) {
+    double value_64f;
+    if (!absl::SimpleAtod(value, &value_64f)) {
+      LOG(ERROR) << "Not a double, expected!";
+      return false;
+    }
+
+    homog_values.push_back(value_64f);
+  }
+
+  return HomographiesFromValues(homog_values, homographies);
+}
+
+bool MotionAnalysisCalculator::HomographiesFromValues(
+    const std::vector<float>& homog_values,
+    std::deque<Homography>* homographies) {
+  CHECK(homographies);
+
+  // Obvious constants are obvious :D
+  constexpr int kHomographyValues = 9;
+  if (homog_values.size() % kHomographyValues != 0) {
+    LOG(ERROR) << "Contents not a multiple of " << kHomographyValues;
+    return false;
+  }
+
+  for (int k = 0; k < homog_values.size(); k += kHomographyValues) {
+    std::vector<double> h_vals(kHomographyValues);
+    for (int l = 0; l < kHomographyValues; ++l) {
+      h_vals[l] = homog_values[k + l];
+    }
+
+    // Normalize last entry to 1.
+    if (h_vals[kHomographyValues - 1] == 0) {
+      LOG(ERROR) << "Degenerate homography, last entry is zero";
+      return false;
+    }
+
+    const double scale = 1.0f / h_vals[kHomographyValues - 1];
+    for (int l = 0; l < kHomographyValues; ++l) {
+      h_vals[l] *= scale;
+    }
+
+    Homography h = HomographyAdapter::FromDoublePointer(h_vals.data(), false);
+    homographies->push_back(h);
+  }
+
+  if (homographies->size() % options_.meta_models_per_frame() != 0) {
+    LOG(ERROR) << "Total homographies not a multiple of specified models "
+               << "per frame.";
+    return false;
+  }
+
+  return true;
+}
+
+void MotionAnalysisCalculator::SubtractMetaMotion(
+    const CameraMotion& meta_motion, RegionFlowFeatureList* features) {
+  if (meta_motion.mixture_homography().model_size() > 0) {
+    CHECK(row_weights_ != nullptr);
+    RegionFlowFeatureListViaTransform(meta_motion.mixture_homography(),
+                                      features, -1.0f,
+                                      1.0f,  // subtract transformed.
+                                      true,  // replace feature loc.
+                                      row_weights_.get());
+  } else {
+    RegionFlowFeatureListViaTransform(meta_motion.homography(), features, -1.0f,
+                                      1.0f,   // subtract transformed.
+                                      true);  // replace feature loc.
+  }
+
+  // Clamp transformed features to domain and handle outliers.
+  const float domain_diam =
+      hypot(features->frame_width(), features->frame_height());
+  const float motion_mag = meta_motion.average_magnitude();
+  // Same irls fraction as used by MODEL_MIXTURE_HOMOGRAPHY scaling in
+  // MotionEstimation.
+  const float irls_fraction = options_.analysis_options()
+                                  .motion_options()
+                                  .irls_mixture_fraction_scale() *
+                              options_.analysis_options()
+                                  .motion_options()
+                                  .irls_motion_magnitude_fraction();
+  float err_scale = std::max(1.0f, motion_mag * irls_fraction);
+
+  const float max_err =
+      options_.meta_outlier_domain_ratio() * domain_diam * err_scale;
+  const float max_err_sq = max_err * max_err;
+
+  for (auto& feature : *features->mutable_feature()) {
+    feature.set_x(
+        std::max(0.0f, std::min(features->frame_width() - 1.0f, feature.x())));
+    feature.set_y(
+        std::max(0.0f, std::min(features->frame_height() - 1.0f, feature.y())));
+    // Label anything with large residual motion an outlier.
+    if (FeatureFlow(feature).Norm2() > max_err_sq) {
+      feature.set_irls_weight(0.0f);
+    }
+  }
+}
+
+void MotionAnalysisCalculator::AddMetaMotion(
+    const CameraMotion& meta_motion, const RegionFlowFeatureList& meta_features,
+    RegionFlowFeatureList* features, CameraMotion* motion) {
+  // Restore old feature location.
+  CHECK_EQ(meta_features.feature_size(), features->feature_size());
+  for (int k = 0; k < meta_features.feature_size(); ++k) {
+    auto feature = features->mutable_feature(k);
+    const auto& meta_feature = meta_features.feature(k);
+    feature->set_x(meta_feature.x());
+    feature->set_y(meta_feature.y());
+    feature->set_dx(meta_feature.dx());
+    feature->set_dy(meta_feature.dy());
+  }
+
+  // Composite camera motion.
+  *motion = ComposeCameraMotion(*motion, meta_motion);
+  // Restore type from metadata, i.e. do not declare motions as invalid.
+  motion->set_type(meta_motion.type());
+  motion->set_match_frame(-1);
+}
+
+void MotionAnalysisCalculator::AppendCameraMotionsFromHomographies(
+    const std::deque<Homography>& homographies, bool append_identity,
+    std::deque<CameraMotion>* camera_motions,
+    std::deque<RegionFlowFeatureList>* features) {
+  CHECK(camera_motions);
+  CHECK(features);
+
+  CameraMotion identity;
+  identity.set_frame_width(frame_width_);
+  identity.set_frame_height(frame_height_);
+
+  *identity.mutable_translation() = TranslationModel();
+  *identity.mutable_linear_similarity() = LinearSimilarityModel();
+  *identity.mutable_homography() = Homography();
+  identity.set_type(CameraMotion::VALID);
+  identity.set_match_frame(0);
+
+  RegionFlowFeatureList empty_list;
+  empty_list.set_long_tracks(true);
+  empty_list.set_match_frame(-1);
+  empty_list.set_frame_width(frame_width_);
+  empty_list.set_frame_height(frame_height_);
+
+  if (append_identity) {
+    camera_motions->push_back(identity);
+    features->push_back(empty_list);
+  }
+
+  const int models_per_frame = options_.meta_models_per_frame();
+  CHECK_GT(models_per_frame, 0) << "At least one model per frame is needed";
+  CHECK_EQ(0, homographies.size() % models_per_frame);
+  const int num_frames = homographies.size() / models_per_frame;
+
+  // Heuristic sigma, similar to what we use for rolling shutter removal.
+  const float mixture_sigma = 1.0f / models_per_frame;
+
+  if (row_weights_ == nullptr) {
+    row_weights_.reset(new MixtureRowWeights(frame_height_,
+                                             frame_height_ / 10,  // 10% margin
+                                             mixture_sigma * frame_height_,
+                                             1.0f, models_per_frame));
+  }
+
+  for (int f = 0; f < num_frames; ++f) {
+    MixtureHomography mix_homog;
+    const int model_start = f * models_per_frame;
+
+    for (int k = 0; k < models_per_frame; ++k) {
+      const Homography& homog = homographies[model_start + k];
+      *mix_homog.add_model() = ModelInvert(homog);
+    }
+
+    CameraMotion c = identity;
+    c.set_match_frame(-1);
+
+    if (mix_homog.model_size() > 1) {
+      *c.mutable_mixture_homography() = mix_homog;
+      c.set_mixture_row_sigma(mixture_sigma);
+
+      for (int k = 0; k < models_per_frame; ++k) {
+        c.add_mixture_inlier_coverage(1.0f);
+      }
+      *c.add_mixture_homography_spectrum() = mix_homog;
+      c.set_rolling_shutter_motion_index(0);
+
+      *c.mutable_homography() = ProjectViaFit<Homography>(
+          mix_homog, frame_width_, frame_height_, row_weights_.get());
+    } else {
+      // Guaranteed to exist because to check that models_per_frame > 0 above.
+      *c.mutable_homography() = mix_homog.model(0);
+    }
+
+    // Project remaining motions down.
+    *c.mutable_linear_similarity() = ProjectViaFit<LinearSimilarityModel>(
+        c.homography(), frame_width_, frame_height_);
+    *c.mutable_translation() = ProjectViaFit<TranslationModel>(
+        c.homography(), frame_width_, frame_height_);
+
+    c.set_average_magnitude(
+        std::hypot(c.translation().dx(), c.translation().dy()));
+
+    camera_motions->push_back(c);
+    features->push_back(empty_list);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/motion_analysis_calculator.proto

@@ -0,0 +1,111 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/motion_analysis.proto";
+
+// Next tag: 10
+message MotionAnalysisCalculatorOptions {
+  extend CalculatorOptions {
+    optional MotionAnalysisCalculatorOptions ext = 270698255;
+  }
+
+  optional mediapipe.MotionAnalysisOptions analysis_options = 1;
+
+  // Determines how optional input SELECTION (if present) is used to compute
+  // the final camera motion.
+  enum SelectionAnalysis {
+    // Recompute camera motion for selected frame neighbors.
+    ANALYSIS_RECOMPUTE = 1;
+
+    // Use composited camera motion and region flow from SELECTION input. No
+    // tracking or re-computation is performed.
+    // Note that in this case only CAMERA, FLOW and VIDEO_OUT tags are
+    // supported as output.
+    NO_ANALYSIS_USE_SELECTION = 2;
+
+    // Recompute camera motion for selected frame neighbors using
+    // features supplied by SELECTION input. No feature tracking is performed.
+    ANALYSIS_FROM_FEATURES = 3;
+
+    // Recomputes camera motion for selected frame neighbors but seeds
+    // initial transform with camera motion from SELECTION input.
+    ANALYSIS_WITH_SEED = 4;
+  }
+
+  optional SelectionAnalysis selection_analysis = 4
+      [default = ANALYSIS_WITH_SEED];
+
+  // If activated when SELECTION input is activated, will replace the computed
+  // camera motion (for any of the ANALYSIS_* case above) with the one supplied
+  // by the frame selection, in case the frame selection one is more stable.
+  // For example, if recomputed camera motion is unstable but the one from
+  // the selection result is stable, will use the stable result instead.
+  optional bool hybrid_selection_camera = 5 [default = false];
+
+  // Determines how optional input META is used to compute the final camera
+  // motion.
+  enum MetaAnalysis {
+    // Uses metadata supplied motions as is.
+    META_ANALYSIS_USE_META = 1;
+
+    // Seeds visual tracking from metadata motions - estimates visual residual
+    // motion and combines with metadata.
+    META_ANALYSIS_HYBRID = 2;
+  }
+
+  optional MetaAnalysis meta_analysis = 8 [default = META_ANALYSIS_USE_META];
+
+  // Determines number of homography models per frame stored in the CSV file
+  // or the homography metadata in META.
+  // For values > 1, MixtureHomographies are created.
+  optional int32 meta_models_per_frame = 6 [default = 1];
+
+  // Used for META_ANALYSIS_HYBRID. Rejects features which flow deviates
+  // domain_ratio * image diagonal size from the ground truth metadata motion.
+  optional float meta_outlier_domain_ratio = 9 [default = 0.0015];
+
+  // If true, the MotionAnalysisCalculator will skip all processing and emit no
+  // packets on any output. This is useful for quickly creating different
+  // versions of a MediaPipe graph without changing its structure, assuming that
+  // downstream calculators can handle missing input packets.
+  // TODO: Remove this hack. See b/36485206 for more details.
+  optional bool bypass_mode = 7 [default = false];
+}
+
+// Taken from
+// java/com/google/android/libraries/microvideo/proto/microvideo.proto to
+// satisfy leakr requirements
+// TODO: Remove and use above proto.
+message HomographyData {
+  // For each frame, there are 12 homography matrices stored. Each matrix is
+  // 3x3 (9 elements). This field will contain 12 x 3 x 3  float values. The
+  // first row of the first homography matrix will be followed by the second row
+  // of the first homography matrix, followed by third row of first homography
+  // matrix, followed by the first row of the second homography matrix, etc.
+  repeated float motion_homography_data = 1 [packed = true];
+
+  // Vector containing histogram counts for individual patches in the frame.
+  repeated uint32 histogram_count_data = 2 [packed = true];
+
+  // The width of the frame at the time metadata was sampled.
+  optional int32 frame_width = 3;
+
+  // The height of the frame at the time metadata was sampled.
+  optional int32 frame_height = 4;
+}

mediapipe/calculators/video/opencv_video_decoder_calculator.cc

@@ -72,6 +72,8 @@ ImageFormat::Format GetImageFormat(int num_channels) {
 // OpenCV's VideoCapture doesn't decode audio tracks. If the audio tracks need
 // to be saved, specify an output side packet with tag "SAVED_AUDIO_PATH".
 // The calculator will call FFmpeg binary to save audio tracks as an aac file.
+// If the audio tracks can't be extracted by FFmpeg, the output side packet
+// will contain an empty std::string.
 //
 // Example config:
 // node {
@@ -150,13 +152,23 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
     if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
 #ifdef HAVE_FFMPEG
       std::string saved_audio_path = std::tmpnam(nullptr);
-      system(absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
+      std::string ffmpeg_command =
-                          " -vn -f adts ", saved_audio_path)
+          absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
-                 .c_str());
+                       " -vn -f adts ", saved_audio_path);
-      cc->OutputSidePackets()
+      system(ffmpeg_command.c_str());
-          .Tag("SAVED_AUDIO_PATH")
+      int status_code = system(absl::StrCat("ls ", saved_audio_path).c_str());
-          .Set(MakePacket<std::string>(saved_audio_path));
+      if (status_code == 0) {
-
+        cc->OutputSidePackets()
+            .Tag("SAVED_AUDIO_PATH")
+            .Set(MakePacket<std::string>(saved_audio_path));
+      } else {
+        LOG(WARNING) << "FFmpeg can't extract audio from " << input_file_path
+                     << " by executing the following command: "
+                     << ffmpeg_command;
+        cc->OutputSidePackets()
+            .Tag("SAVED_AUDIO_PATH")
+            .Set(MakePacket<std::string>(std::string()));
+      }
 #else
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
              << "OpenCVVideoDecoderCalculator can't save the audio file "

mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc

@@ -55,8 +55,12 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
   EXPECT_EQ(640, header.height);
   EXPECT_FLOAT_EQ(6.0f, header.duration);
   EXPECT_FLOAT_EQ(30.0f, header.frame_rate);
-  EXPECT_EQ(180, runner.Outputs().Tag("VIDEO").packets.size());
+  // The number of the output packets should be 180.
-  for (int i = 0; i < 180; ++i) {
+  // Some OpenCV version returns the first two frames with the same timestamp on
+  // macos and we might miss one frame here.
+  int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size();
+  EXPECT_GE(num_of_packets, 179);
+  for (int i = 0; i < num_of_packets; ++i) {
     Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i];
     cv::Mat output_mat =
         formats::MatView(&(image_frame_packet.Get<ImageFrame>()));
@@ -141,8 +145,12 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMkvVp8Video) {
   EXPECT_EQ(320, header.height);
   EXPECT_FLOAT_EQ(6.0f, header.duration);
   EXPECT_FLOAT_EQ(30.0f, header.frame_rate);
-  EXPECT_EQ(180, runner.Outputs().Tag("VIDEO").packets.size());
+  // The number of the output packets should be 180.
-  for (int i = 0; i < 180; ++i) {
+  // Some OpenCV version returns the first two frames with the same timestamp on
+  // macos and we might miss one frame here.
+  int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size();
+  EXPECT_GE(num_of_packets, 179);
+  for (int i = 0; i < num_of_packets; ++i) {
     Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i];
     cv::Mat output_mat =
         formats::MatView(&(image_frame_packet.Get<ImageFrame>()));

mediapipe/calculators/video/opencv_video_encoder_calculator.cc

@@ -183,14 +183,20 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
 #ifdef HAVE_FFMPEG
     const std::string& audio_file_path =
         cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Get<std::string>();
-    // A temp output file is needed because FFmpeg can't do in-place editing.
+    if (audio_file_path.empty()) {
-    const std::string temp_file_path = std::tmpnam(nullptr);
+      LOG(WARNING) << "OpenCvVideoEncoderCalculator isn't able to attach the "
-    system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
+                      "audio tracks to the generated video because the audio "
-                        "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
+                      "file path is not specified.";
-                        " -i ", audio_file_path,
+    } else {
-                        "  -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
+      // A temp output file is needed because FFmpeg can't do in-place editing.
-                        "&& rm ", temp_file_path)
+      const std::string temp_file_path = std::tmpnam(nullptr);
-               .c_str());
+      system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
+                          "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
+                          " -i ", audio_file_path,
+                          "  -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
+                          "&& rm ", temp_file_path)
+                 .c_str());
+    }
 
 #else
     return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)

mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc

@@ -210,8 +210,8 @@ TEST(OpenCvVideoEncoderCalculatorTest, TestMkvVp8Video) {
   EXPECT_EQ(video_header.frame_rate,
             static_cast<double>(cap.get(cv::CAP_PROP_FPS)));
   EXPECT_EQ(video_header.duration,
-            static_cast<int>(cap.get(cv::CAP_PROP_FRAME_COUNT) /
+            static_cast<int>(std::round(cap.get(cv::CAP_PROP_FRAME_COUNT) /
-                             cap.get(cv::CAP_PROP_FPS)));
+                                        cap.get(cv::CAP_PROP_FPS))));
 }
 
 }  // namespace

mediapipe/calculators/video/testdata/parallel_tracker_graph.pbtxt

@@ -0,0 +1,134 @@
+input_stream: "image_cpu_frames"
+input_stream: "start_pos"
+input_stream: "ra_track"
+
+num_threads: 4
+
+node: {
+  calculator: "MotionAnalysisCalculator"
+  input_stream: "VIDEO:image_cpu_frames"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+
+  options: {
+    [mediapipe.MotionAnalysisCalculatorOptions.ext]: {
+      analysis_options: {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+
+        flow_options: {
+          # Maybe move down to 50
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          downsample_mode: DOWNSAMPLE_NONE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options: {
+            max_features: 500
+            adaptive_extraction_levels: 2
+            min_eig_val_settings: {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+
+        motion_options: {
+          label_empty_frames_as_valid: false
+        }
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "FlowPackagerCalculator"
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+
+  options: {
+    [mediapipe.FlowPackagerCalculatorOptions.ext]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "START_POS:start_pos"
+  output_stream: "BOXES:boxes"
+  input_side_packet: "OPTIONS:calculator_options"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+        }
+        sync_set: {
+          tag_index: "START_POS"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE
+          object_similarity_min_contd_inliers: 6
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "RA_TRACK:ra_track"
+  output_stream: "RA_BOXES:ra_boxes"
+  input_side_packet: "OPTIONS:calculator_options"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+        }
+        sync_set: {
+          tag_index: "RA_TRACK"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE
+          object_similarity_min_contd_inliers: 6
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}

mediapipe/calculators/video/testdata/tracker_graph.pbtxt

@@ -0,0 +1,122 @@
+input_stream: "image_cpu_frames"
+input_stream: "start_pos"
+input_stream: "cancel_object_id"
+input_stream: "ra_track"
+input_stream: "restart_pos"
+input_stream: "track_time"
+
+num_threads: 4
+
+node: {
+  calculator: "MotionAnalysisCalculator"
+  options: {
+    [mediapipe.MotionAnalysisCalculatorOptions.ext]: {
+      analysis_options: {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+
+        flow_options: {
+          # Maybe move down to 50
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          # For mobile application, downsample before input into graph
+          # and use DOWNSAMPLE_TO_INPUT_SIZE and specify
+          # downsampling_factor option or DOWNSAMPLE input_side_packet
+          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options: {
+            max_features: 500
+            corner_extraction_method: EXTRACTION_FAST
+            adaptive_extraction_levels: 2
+            min_eig_val_settings: {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+      }
+    }
+  }
+
+  # Drops packets if calculator cannot keep up with the input rate.
+  input_stream_handler: {
+    input_stream_handler: "FixedSizeInputStreamHandler"
+  }
+
+  input_stream: "VIDEO:image_cpu_frames"
+  input_side_packet: "DOWNSAMPLE:analysis_downsample_factor"
+  input_side_packet: "OPTIONS:calculator_options"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+}
+
+node: {
+  calculator: "FlowPackagerCalculator"
+
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+  options: {
+    [mediapipe.FlowPackagerCalculatorOptions.ext]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+node: {
+  calculator: "BoxTrackerCalculator"
+
+  input_side_packet: "OPTIONS:calculator_options"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "TRACK_TIME:track_time"
+  input_stream: "START_POS:start_pos"
+  input_stream: "RESTART_POS:restart_pos"
+  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  input_stream: "RA_TRACK:ra_track"
+  output_stream: "BOXES:boxes"
+  output_stream: "RA_BOXES:ra_boxes"
+
+  input_stream_handler: {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options: {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext]: {
+        sync_set: {
+          tag_index: "TRACKING"
+          tag_index: "TRACK_TIME"
+        }
+        sync_set: {
+          tag_index: "START_POS"
+        }
+        sync_set: {
+          tag_index: "RESTART_POS"
+        }
+        sync_set: {
+          tag_index: "CANCEL_OBJECT_ID"
+        }
+        sync_set: {
+          tag_index: "RA_TRACK"
+        }
+      }
+    }
+  }
+
+  options: {
+    [mediapipe.BoxTrackerCalculatorOptions.ext]: {
+      tracker_options: {
+        track_step_options: {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+          object_similarity_min_contd_inliers: 10
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}

mediapipe/calculators/video/tracked_detection_manager_calculator.cc

@@ -0,0 +1,319 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "absl/container/node_hash_map.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+#include "mediapipe/util/tracking/tracked_detection.h"
+#include "mediapipe/util/tracking/tracked_detection_manager.h"
+#include "mediapipe/util/tracking/tracking.h"
+
+namespace mediapipe {
+namespace {
+
+constexpr int kDetectionUpdateTimeOutMS = 5000;
+constexpr char kDetectionsTag[] = "DETECTIONS";
+constexpr char kDetectionBoxesTag[] = "DETECTION_BOXES";
+constexpr char kDetectionListTag[] = "DETECTION_LIST";
+constexpr char kTrackingBoxesTag[] = "TRACKING_BOXES";
+constexpr char kCancelObjectIdTag[] = "CANCEL_OBJECT_ID";
+
+// Move |src| to the back of |dst|.
+void MoveIds(std::vector<int>* dst, std::vector<int> src) {
+  dst->insert(dst->end(), std::make_move_iterator(src.begin()),
+              std::make_move_iterator(src.end()));
+}
+
+int64 GetInputTimestampMs(::mediapipe::CalculatorContext* cc) {
+  return cc->InputTimestamp().Microseconds() / 1000;  // 1 ms = 1000 us.
+}
+
+// Converts a Mediapipe Detection Proto to a TrackedDetection class.
+std::unique_ptr<TrackedDetection> GetTrackedDetectionFromDetection(
+    const Detection& detection, int64 timestamp) {
+  std::unique_ptr<TrackedDetection> tracked_detection =
+      absl::make_unique<TrackedDetection>(detection.detection_id(), timestamp);
+  const float top = detection.location_data().relative_bounding_box().ymin();
+  const float bottom =
+      detection.location_data().relative_bounding_box().ymin() +
+      detection.location_data().relative_bounding_box().height();
+  const float left = detection.location_data().relative_bounding_box().xmin();
+  const float right = detection.location_data().relative_bounding_box().xmin() +
+                      detection.location_data().relative_bounding_box().width();
+  NormalizedRect bounding_box;
+  bounding_box.set_x_center((left + right) / 2.f);
+  bounding_box.set_y_center((top + bottom) / 2.f);
+  bounding_box.set_height(bottom - top);
+  bounding_box.set_width(right - left);
+  tracked_detection->set_bounding_box(bounding_box);
+
+  for (int i = 0; i < detection.label_size(); ++i) {
+    tracked_detection->AddLabel(detection.label(i), detection.score(i));
+  }
+  return tracked_detection;
+}
+
+// Converts a TrackedDetection class to a Mediapipe Detection Proto.
+Detection GetAxisAlignedDetectionFromTrackedDetection(
+    const TrackedDetection& tracked_detection) {
+  Detection detection;
+  LocationData* location_data = detection.mutable_location_data();
+
+  auto corners = tracked_detection.GetCorners();
+
+  float x_min = std::numeric_limits<float>::max();
+  float x_max = std::numeric_limits<float>::min();
+  float y_min = std::numeric_limits<float>::max();
+  float y_max = std::numeric_limits<float>::min();
+  for (int i = 0; i < 4; ++i) {
+    x_min = std::min(x_min, corners[i].x());
+    x_max = std::max(x_max, corners[i].x());
+    y_min = std::min(y_min, corners[i].y());
+    y_max = std::max(y_max, corners[i].y());
+  }
+  location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);
+  LocationData::RelativeBoundingBox* relative_bbox =
+      location_data->mutable_relative_bounding_box();
+  relative_bbox->set_xmin(x_min);
+  relative_bbox->set_ymin(y_min);
+  relative_bbox->set_width(x_max - x_min);
+  relative_bbox->set_height(y_max - y_min);
+
+  // Use previous id which is the id the object when it's first detected.
+  if (tracked_detection.previous_id() > 0) {
+    detection.set_detection_id(tracked_detection.previous_id());
+  } else {
+    detection.set_detection_id(tracked_detection.unique_id());
+  }
+  for (const auto& label_and_score : tracked_detection.label_to_score_map()) {
+    detection.add_label(label_and_score.first);
+    detection.add_score(label_and_score.second);
+  }
+  return detection;
+}
+
+}  // namespace
+
+// TrackedDetectionManagerCalculator accepts detections and tracking results at
+// different frame rate for real time tracking of targets.
+// Input:
+//   DETECTIONS: A vector<Detection> of newly detected targets.
+//   TRACKING_BOXES: A TimedBoxProtoList which contains a list of tracked boxes
+//   from previous detections.
+//
+// Output:
+//   CANCEL_OBJECT_ID: Ids of targets that are missing/lost such that it should
+//   be removed from tracking.
+//   DETECTIONS: List of detections that are being tracked.
+//   DETECTION_BOXES: List of bounding boxes of detections that are being
+//   tracked.
+//
+// Usage example:
+// node {
+//   calculator: "TrackedDetectionManagerCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   input_stream: "TRACKING_BOXES:boxes"
+//   output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+//   output_stream: "DETECTIONS:output_detections"
+// }
+class TrackedDetectionManagerCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Adds new list of detections to |waiting_for_update_detections_|.
+  void AddDetectionList(const DetectionList& detection_list,
+                        CalculatorContext* cc);
+  void AddDetections(const std::vector<Detection>& detections,
+                     CalculatorContext* cc);
+
+  // Manages existing and new detections.
+  TrackedDetectionManager tracked_detection_manager_;
+
+  // Set of detections that are not up to date yet. These detections will be
+  // added to the detection manager until they got updated from the box tracker.
+  absl::node_hash_map<int, std::unique_ptr<TrackedDetection>>
+      waiting_for_update_detections_;
+};
+REGISTER_CALCULATOR(TrackedDetectionManagerCalculator);
+
+::mediapipe::Status TrackedDetectionManagerCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag(kDetectionsTag)) {
+    cc->Inputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+  }
+  if (cc->Inputs().HasTag(kDetectionListTag)) {
+    cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
+  }
+  if (cc->Inputs().HasTag(kTrackingBoxesTag)) {
+    cc->Inputs().Tag(kTrackingBoxesTag).Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Outputs().HasTag(kCancelObjectIdTag)) {
+    cc->Outputs().Tag(kCancelObjectIdTag).Set<int>();
+  }
+  if (cc->Outputs().HasTag(kDetectionsTag)) {
+    cc->Outputs().Tag(kDetectionsTag).Set<std::vector<Detection>>();
+  }
+  if (cc->Outputs().HasTag(kDetectionBoxesTag)) {
+    cc->Outputs().Tag(kDetectionBoxesTag).Set<std::vector<NormalizedRect>>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TrackedDetectionManagerCalculator::Process(
+    CalculatorContext* cc) {
+  if (cc->Inputs().HasTag("TRACKING_BOXES")) {
+    if (!cc->Inputs().Tag("TRACKING_BOXES").IsEmpty()) {
+      const TimedBoxProtoList& tracked_boxes =
+          cc->Inputs().Tag("TRACKING_BOXES").Get<TimedBoxProtoList>();
+
+      // Collect all detections that are removed.
+      auto removed_detection_ids = absl::make_unique<std::vector<int>>();
+      for (const TimedBoxProto& tracked_box : tracked_boxes.box()) {
+        NormalizedRect bounding_box;
+        bounding_box.set_x_center((tracked_box.left() + tracked_box.right()) /
+                                  2.f);
+        bounding_box.set_y_center((tracked_box.bottom() + tracked_box.top()) /
+                                  2.f);
+        bounding_box.set_height(tracked_box.bottom() - tracked_box.top());
+        bounding_box.set_width(tracked_box.right() - tracked_box.left());
+        bounding_box.set_rotation(tracked_box.rotation());
+        // First check if this box updates a detection that's waiting for
+        // update from the tracker.
+        auto waiting_for_update_detectoin_ptr =
+            waiting_for_update_detections_.find(tracked_box.id());
+        if (waiting_for_update_detectoin_ptr !=
+            waiting_for_update_detections_.end()) {
+          // Add the detection and remove duplicated detections.
+          auto removed_ids = tracked_detection_manager_.AddDetection(
+              std::move(waiting_for_update_detectoin_ptr->second));
+          MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+          waiting_for_update_detections_.erase(
+              waiting_for_update_detectoin_ptr);
+        }
+        auto removed_ids = tracked_detection_manager_.UpdateDetectionLocation(
+            tracked_box.id(), bounding_box, tracked_box.time_msec());
+        MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+      }
+      // TODO: Should be handled automatically in detection manager.
+      auto removed_ids = tracked_detection_manager_.RemoveObsoleteDetections(
+          GetInputTimestampMs(cc) - kDetectionUpdateTimeOutMS);
+      MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+      // TODO: Should be handled automatically in detection manager.
+      removed_ids = tracked_detection_manager_.RemoveOutOfViewDetections();
+      MoveIds(removed_detection_ids.get(), std::move(removed_ids));
+
+      if (!removed_detection_ids->empty() &&
+          cc->Outputs().HasTag(kCancelObjectIdTag)) {
+        auto timestamp = cc->InputTimestamp();
+        for (int box_id : *removed_detection_ids) {
+          // The timestamp is incremented (by 1 us) because currently the box
+          // tracker calculator only accepts one cancel object ID for any given
+          // timestamp.
+          cc->Outputs()
+              .Tag(kCancelObjectIdTag)
+              .AddPacket(mediapipe::MakePacket<int>(box_id).At(timestamp++));
+        }
+      }
+
+      // Output detections and corresponding bounding boxes.
+      const auto& all_detections =
+          tracked_detection_manager_.GetAllTrackedDetections();
+      auto output_detections = absl::make_unique<std::vector<Detection>>();
+      auto output_boxes = absl::make_unique<std::vector<NormalizedRect>>();
+
+      for (const auto& detection_ptr : all_detections) {
+        const auto& detection = *detection_ptr.second;
+        // Only output detections that are synced.
+        if (detection.last_updated_timestamp() <
+            cc->InputTimestamp().Microseconds() / 1000) {
+          continue;
+        }
+        output_detections->emplace_back(
+            GetAxisAlignedDetectionFromTrackedDetection(detection));
+        output_boxes->emplace_back(detection.bounding_box());
+      }
+      if (cc->Outputs().HasTag(kDetectionsTag)) {
+        cc->Outputs()
+            .Tag(kDetectionsTag)
+            .Add(output_detections.release(), cc->InputTimestamp());
+      }
+
+      if (cc->Outputs().HasTag(kDetectionBoxesTag)) {
+        cc->Outputs()
+            .Tag(kDetectionBoxesTag)
+            .Add(output_boxes.release(), cc->InputTimestamp());
+      }
+    }
+  }
+
+  if (cc->Inputs().HasTag(kDetectionsTag) &&
+      !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) {
+    const auto detections =
+        cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>();
+    AddDetections(detections, cc);
+  }
+
+  if (cc->Inputs().HasTag(kDetectionListTag) &&
+      !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) {
+    const auto detection_list =
+        cc->Inputs().Tag(kDetectionListTag).Get<DetectionList>();
+    AddDetectionList(detection_list, cc);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void TrackedDetectionManagerCalculator::AddDetectionList(
+    const DetectionList& detection_list, CalculatorContext* cc) {
+  for (const auto& detection : detection_list.detection()) {
+    // Convert from microseconds to milliseconds.
+    std::unique_ptr<TrackedDetection> new_detection =
+        GetTrackedDetectionFromDetection(
+            detection, cc->InputTimestamp().Microseconds() / 1000);
+
+    const int id = new_detection->unique_id();
+    waiting_for_update_detections_[id] = std::move(new_detection);
+  }
+}
+
+void TrackedDetectionManagerCalculator::AddDetections(
+    const std::vector<Detection>& detections, CalculatorContext* cc) {
+  for (const auto& detection : detections) {
+    // Convert from microseconds to milliseconds.
+    std::unique_ptr<TrackedDetection> new_detection =
+        GetTrackedDetectionFromDetection(
+            detection, cc->InputTimestamp().Microseconds() / 1000);
+
+    const int id = new_detection->unique_id();
+    waiting_for_update_detections_[id] = std::move(new_detection);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/tracking_graph_test.cc

@@ -0,0 +1,709 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/calculators/video/box_tracker_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/tracking.pb.h"
+
+#ifdef __APPLE__
+#include <CoreFoundation/CoreFoundation.h>
+#endif  // defined(__APPLE__)
+
+namespace mediapipe {
+namespace {
+using ::testing::FloatNear;
+using ::testing::Test;
+
+std::string GetTestDir() {
+#ifdef __APPLE__
+  char path[1024];
+  CFURLRef bundle_url = CFBundleCopyBundleURL(CFBundleGetMainBundle());
+  CFURLGetFileSystemRepresentation(
+      bundle_url, true, reinterpret_cast<UInt8*>(path), sizeof(path));
+  CFRelease(bundle_url);
+  return ::mediapipe::file::JoinPath(path, "testdata");
+#elif defined(__ANDROID__)
+  char path[1024];
+  getcwd(path, sizeof(path));
+  return ::mediapipe::file::JoinPath(path,
+                                     "mediapipe/calculators/video/testdata");
+#else
+  return ::mediapipe::file::JoinPath(
+      "./",
+      // This should match the path of the output files
+      // of the genrule() that generates test model files.
+      "mediapipe/calculators/video/testdata");
+#endif  // defined(__APPLE__)
+}
+
+bool LoadBinaryTestGraph(const std::string& graph_path,
+                         CalculatorGraphConfig* config) {
+  std::ifstream ifs;
+  ifs.open(graph_path.c_str());
+  proto_ns::io::IstreamInputStream in_stream(&ifs);
+  bool success = config->ParseFromZeroCopyStream(&in_stream);
+  ifs.close();
+  if (!success) {
+    LOG(ERROR) << "could not parse test graph: " << graph_path;
+  }
+  return success;
+}
+
+class TrackingGraphTest : public Test {
+ protected:
+  TrackingGraphTest() {}
+
+  void SetUp() override {
+    test_dir_ = GetTestDir();
+    const auto graph_path = file::JoinPath(test_dir_, "tracker.binarypb");
+    ASSERT_TRUE(LoadBinaryTestGraph(graph_path, &config_));
+
+    original_image_ = cv::imread(file::JoinPath(test_dir_, "lenna.png"));
+    CreateInputFramesFromOriginalImage(kNumImages, kTranslationStep,
+                                       &input_frames_packets_);
+
+    const auto& first_input_img = input_frames_packets_[0].Get<ImageFrame>();
+    const int img_width = first_input_img.Width();
+    const int img_height = first_input_img.Height();
+    translation_step_x_ = kTranslationStep / static_cast<float>(img_width);
+    translation_step_y_ = kTranslationStep / static_cast<float>(img_height);
+
+    // Creat new configure and packet dump vector to store output.
+    mediapipe::CalculatorGraphConfig config_copy = config_;
+    mediapipe::tool::AddVectorSink("boxes", &config_copy, &output_packets_);
+    mediapipe::tool::AddVectorSink("ra_boxes", &config_copy,
+                                   &random_access_results_packets_);
+
+    // Initialize graph
+    MP_ASSERT_OK(graph_.Initialize(config_copy));
+
+    const auto parallel_graph_path =
+        file::JoinPath(test_dir_, "parallel_tracker.binarypb");
+    CalculatorGraphConfig parallel_config;
+    ASSERT_TRUE(LoadBinaryTestGraph(parallel_graph_path, &parallel_config));
+    mediapipe::tool::AddVectorSink("boxes", &parallel_config, &output_packets_);
+    mediapipe::tool::AddVectorSink("ra_boxes", &parallel_config,
+                                   &random_access_results_packets_);
+    MP_ASSERT_OK(parallel_graph_.Initialize(parallel_config));
+  }
+
+  void CreateInputFramesFromOriginalImage(
+      int num_images, int translation_step,
+      std::vector<Packet>* input_frames_packets);
+
+  void TearDown() override {
+    output_packets_.clear();
+    random_access_results_packets_.clear();
+  }
+
+  std::unique_ptr<TimedBoxProtoList> MakeBoxList(
+      const Timestamp& timestamp, const std::vector<bool>& is_quad_tracking,
+      const std::vector<bool>& is_pnp_tracking,
+      const std::vector<bool>& reacquisition) const;
+
+  void RunGraphWithSidePacketsAndInputs(
+      const std::map<std::string, mediapipe::Packet>& side_packets,
+      const mediapipe::Packet& start_pos_packet);
+
+  // Utility functions used to judge if a given quad or box is near to the
+  // groundtruth location at a given frame.
+  // Examine box.reacquisition() field equals to `reacquisition`.
+  // `frame` can be float number to account for inter-frame interpolation.
+  void ExpectBoxAtFrame(const TimedBoxProto& box, float frame,
+                        bool reacquisition);
+
+  // Examine box.aspect_ratio() field equals to `aspect_ratio` if asepct_ratio
+  // is positive.
+  void ExpectQuadAtFrame(const TimedBoxProto& box, float frame,
+                         float aspect_ratio, bool reacquisition);
+
+  // Utility function to judge if two quad are near to each other.
+  void ExpectQuadNear(const TimedBoxProto& box1, const TimedBoxProto& box2);
+
+  std::unique_ptr<TimedBoxProtoList> CreateRandomAccessTrackingBoxList(
+      const std::vector<Timestamp>& start_timestamps,
+      const std::vector<Timestamp>& end_timestamps) const;
+
+  CalculatorGraph graph_;
+  CalculatorGraph parallel_graph_;
+  CalculatorGraphConfig config_;
+  std::string test_dir_;
+  cv::Mat original_image_;
+  std::vector<Packet> input_frames_packets_;
+  std::vector<mediapipe::Packet> output_packets_;
+  std::vector<mediapipe::Packet> random_access_results_packets_;
+  float translation_step_x_;  // normalized translation step in x direction
+  float translation_step_y_;  // normalized translation step in y direction
+  static constexpr float kInitialBoxHalfWidthNormalized = 0.25f;
+  static constexpr float kInitialBoxHalfHeightNormalized = 0.25f;
+  static constexpr float kImageAspectRatio = 1.0f;  // for lenna.png
+  static constexpr float kInitialBoxLeft =
+      0.5f - kInitialBoxHalfWidthNormalized;
+  static constexpr float kInitialBoxRight =
+      0.5f + kInitialBoxHalfWidthNormalized;
+  static constexpr float kInitialBoxTop =
+      0.5f - kInitialBoxHalfHeightNormalized;
+  static constexpr float kInitialBoxBottom =
+      0.5f + kInitialBoxHalfHeightNormalized;
+  static constexpr int kFrameIntervalUs = 30000;
+  static constexpr int kNumImages = 8;
+  // Each image is shifted to the right and bottom by kTranslationStep
+  // pixels compared with the previous image.
+  static constexpr int kTranslationStep = 10;
+  static constexpr float kEqualityTolerance = 3e-4f;
+};
+
+void TrackingGraphTest::ExpectBoxAtFrame(const TimedBoxProto& box, float frame,
+                                         bool reacquisition) {
+  EXPECT_EQ(box.reacquisition(), reacquisition);
+  EXPECT_TRUE(box.has_rotation());
+  EXPECT_THAT(box.rotation(), FloatNear(0, kEqualityTolerance));
+  EXPECT_THAT(box.left(),
+              FloatNear(kInitialBoxLeft - frame * translation_step_x_,
+                        kEqualityTolerance));
+  EXPECT_THAT(box.top(), FloatNear(kInitialBoxTop - frame * translation_step_y_,
+                                   kEqualityTolerance));
+  EXPECT_THAT(box.bottom(),
+              FloatNear(kInitialBoxBottom - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(box.right(),
+              FloatNear(kInitialBoxRight - frame * translation_step_x_,
+                        kEqualityTolerance));
+}
+
+void TrackingGraphTest::ExpectQuadAtFrame(const TimedBoxProto& box, float frame,
+                                          float aspect_ratio,
+                                          bool reacquisition) {
+  EXPECT_TRUE(box.has_quad()) << "quad must exist!";
+  if (aspect_ratio > 0) {
+    EXPECT_TRUE(box.has_aspect_ratio());
+    EXPECT_NEAR(box.aspect_ratio(), aspect_ratio, kEqualityTolerance);
+  }
+
+  EXPECT_EQ(box.reacquisition(), reacquisition);
+
+  const auto& quad = box.quad();
+  EXPECT_EQ(8, quad.vertices_size())
+      << "quad has only " << box.quad().vertices_size() << " vertices";
+  EXPECT_THAT(quad.vertices(0),
+              FloatNear(kInitialBoxLeft - frame * translation_step_x_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(1),
+              FloatNear(kInitialBoxTop - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(3),
+              FloatNear(kInitialBoxBottom - frame * translation_step_y_,
+                        kEqualityTolerance));
+  EXPECT_THAT(quad.vertices(4),
+              FloatNear(kInitialBoxRight - frame * translation_step_x_,
+                        kEqualityTolerance));
+}
+
+void TrackingGraphTest::ExpectQuadNear(const TimedBoxProto& box1,
+                                       const TimedBoxProto& box2) {
+  EXPECT_TRUE(box1.has_quad());
+  EXPECT_TRUE(box2.has_quad());
+  EXPECT_EQ(8, box1.quad().vertices_size())
+      << "quad has only " << box1.quad().vertices_size() << " vertices";
+  EXPECT_EQ(8, box2.quad().vertices_size())
+      << "quad has only " << box2.quad().vertices_size() << " vertices";
+  for (int j = 0; j < box1.quad().vertices_size(); ++j) {
+    EXPECT_NEAR(box1.quad().vertices(j), box2.quad().vertices(j),
+                kEqualityTolerance);
+  }
+}
+
+std::unique_ptr<TimedBoxProtoList> TrackingGraphTest::MakeBoxList(
+    const Timestamp& timestamp, const std::vector<bool>& is_quad_tracking,
+    const std::vector<bool>& is_pnp_tracking,
+    const std::vector<bool>& reacquisition) const {
+  auto box_list = absl::make_unique<TimedBoxProtoList>();
+  int box_id = 0;
+  for (int j = 0; j < is_quad_tracking.size(); ++j) {
+    TimedBoxProto* box = box_list->add_box();
+    if (is_quad_tracking[j]) {
+      box->mutable_quad()->add_vertices(kInitialBoxLeft);
+      box->mutable_quad()->add_vertices(kInitialBoxTop);
+      box->mutable_quad()->add_vertices(kInitialBoxLeft);
+      box->mutable_quad()->add_vertices(kInitialBoxBottom);
+      box->mutable_quad()->add_vertices(kInitialBoxRight);
+      box->mutable_quad()->add_vertices(kInitialBoxBottom);
+      box->mutable_quad()->add_vertices(kInitialBoxRight);
+      box->mutable_quad()->add_vertices(kInitialBoxTop);
+
+      if (is_pnp_tracking[j]) {
+        box->set_aspect_ratio(kImageAspectRatio);
+      }
+    } else {
+      box->set_left(kInitialBoxLeft);
+      box->set_right(kInitialBoxRight);
+      box->set_top(kInitialBoxTop);
+      box->set_bottom(kInitialBoxBottom);
+    }
+    box->set_id(box_id++);
+    box->set_time_msec(timestamp.Value() / 1000);
+    box->set_reacquisition(reacquisition[j]);
+  }
+
+  return box_list;
+}
+
+void TrackingGraphTest::CreateInputFramesFromOriginalImage(
+    int num_images, int translation_step,
+    std::vector<Packet>* input_frames_packets) {
+  const int crop_width = original_image_.cols - num_images * translation_step;
+  const int crop_height = original_image_.rows - num_images * translation_step;
+  for (int i = 0; i < num_images; ++i) {
+    cv::Rect roi(i * translation_step, i * translation_step, crop_width,
+                 crop_height);
+    cv::Mat cropped_img = cv::Mat(original_image_, roi);
+    auto cropped_image_frame = absl::make_unique<ImageFrame>(
+        ImageFormat::SRGB, crop_width, crop_height, cropped_img.step[0],
+        cropped_img.data, ImageFrame::PixelDataDeleter::kNone);
+    Timestamp curr_timestamp = Timestamp(i * kFrameIntervalUs);
+    Packet image_packet =
+        Adopt(cropped_image_frame.release()).At(curr_timestamp);
+    input_frames_packets->push_back(image_packet);
+  }
+}
+
+void TrackingGraphTest::RunGraphWithSidePacketsAndInputs(
+    const std::map<std::string, mediapipe::Packet>& side_packets,
+    const mediapipe::Packet& start_pos_packet) {
+  // Start running the graph
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet));
+
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("image_cpu_frames", frame_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
+std::unique_ptr<TimedBoxProtoList>
+TrackingGraphTest::CreateRandomAccessTrackingBoxList(
+    const std::vector<Timestamp>& start_timestamps,
+    const std::vector<Timestamp>& end_timestamps) const {
+  CHECK_EQ(start_timestamps.size(), end_timestamps.size());
+  auto ra_boxes = absl::make_unique<TimedBoxProtoList>();
+  for (int i = 0; i < start_timestamps.size(); ++i) {
+    auto start_box_list =
+        MakeBoxList(start_timestamps[i], std::vector<bool>{true},
+                    std::vector<bool>{true}, std::vector<bool>{false});
+    auto end_box_list =
+        MakeBoxList(end_timestamps[i], std::vector<bool>{true},
+                    std::vector<bool>{true}, std::vector<bool>{false});
+    *(ra_boxes->add_box()) = (*start_box_list).box(0);
+    *(ra_boxes->add_box()) = (*end_box_list).box(0);
+  }
+  return ra_boxes;
+}
+
+TEST_F(TrackingGraphTest, BasicBoxTrackingSanityCheck) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(CalculatorOptions())));
+
+  // Run the graph with input side packets, start_pos, and input image frames.
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // is_quad_tracking is used to indicate whether to track quad for each
+  // individual box.
+  std::vector<bool> is_quad_tracking{false};
+  // is_pnp_tracking is used to indicate whether to use perspective transform to
+  // track quad.
+  std::vector<bool> is_pnp_tracking{false};
+  // is_reacquisition is used to indicate whether to enable reacquisition for
+  // the box.
+  std::vector<bool> is_reacquisition{false};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+  RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet);
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    ExpectBoxAtFrame(boxes.box(0), i, false);
+  }
+}
+
+TEST_F(TrackingGraphTest, BasicQuadTrackingSanityCheck) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // Box id 0 use quad tracking with 8DoF homography transform.
+  // Box id 1 use quad tracking with 6DoF perspective transform.
+  // Box id 2 use box tracking with 4DoF similarity transform.
+  std::vector<bool> is_quad_tracking{true, true, false};
+  std::vector<bool> is_pnp_tracking{false, true, false};
+  std::vector<bool> is_reacquisition{true, false, true};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+  RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet);
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    for (int j = 0; j < boxes.box_size(); ++j) {
+      const TimedBoxProto& box = boxes.box(j);
+      if (is_quad_tracking[box.id()]) {
+        ExpectQuadAtFrame(box, i,
+                          is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f,
+                          is_reacquisition[box.id()]);
+      } else {
+        ExpectBoxAtFrame(box, i, is_reacquisition[box.id()]);
+      }
+    }
+  }
+}
+
+TEST_F(TrackingGraphTest, TestRandomAccessTrackingResults) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  ASSERT_GT(input_frames_packets_.size(), 2);  // at least 3 frames
+  ASSERT_TRUE(input_frames_packets_[2].Timestamp() -
+                  input_frames_packets_[1].Timestamp() >
+              TimestampDiff(1000));
+
+  constexpr int start_frame = 0;
+  Timestamp start_box_time = input_frames_packets_[start_frame].Timestamp();
+  auto start_box_list =
+      MakeBoxList(start_box_time, std::vector<bool>{true},
+                  std::vector<bool>{true}, std::vector<bool>{false});
+  constexpr int end_frame = 2;
+  Timestamp end_box_time = input_frames_packets_[end_frame].Timestamp();
+
+  // Also test reverse random access tracking.
+  // This offset of 1ms is simulating the case where the start query timestamp
+  // to be not any existing frame timestamp. In reality, it's highly encouraged
+  // to have the start query timestamp be aligned with frame timestamp.
+  constexpr int reverse_start_frame = 1;
+  Timestamp reverse_start_box_time =
+      input_frames_packets_[reverse_start_frame].Timestamp() + 1000;
+
+  auto ra_boxes = CreateRandomAccessTrackingBoxList(
+      {start_box_time, reverse_start_box_time}, {end_box_time, start_box_time});
+
+  Packet ra_packet = Adopt(ra_boxes.release()).At(start_box_time);
+  Packet start_packet = Adopt(start_box_list.release()).At(start_box_time);
+
+  // Start running the ordinary graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_packet));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("image_cpu_frames", frame_packet));
+    Packet track_time_packet = Adopt(new int(0)).At(frame_packet.Timestamp());
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("track_time", track_time_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  const TimedBoxProtoList tracking_result =
+      output_packets_[end_frame].Get<TimedBoxProtoList>();
+  EXPECT_EQ(1, tracking_result.box_size());
+
+  // Should have 1 random access packet.
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const TimedBoxProtoList& ra_result =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  // Two box tracking results. One for comparison with normal tracking. The
+  // other for reverse random access tracking.
+  EXPECT_EQ(2, ra_result.box_size());
+
+  // Check if randan access tracking has same result with normal tracking.
+  ExpectQuadNear(tracking_result.box(0), ra_result.box(0));
+  ExpectQuadAtFrame(ra_result.box(0), end_frame - start_frame,
+                    kImageAspectRatio, false);
+  ExpectQuadAtFrame(ra_result.box(1), start_frame - reverse_start_frame - 1,
+                    kImageAspectRatio, false);
+
+  // Clear output and ra result packet vector before test parallel graph.
+  TearDown();
+
+  // Start running the parallel graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(parallel_graph_.StartRun(side_packets));
+  MP_EXPECT_OK(
+      parallel_graph_.AddPacketToInputStream("start_pos", start_packet));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames",
+                                                        frame_packet));
+    MP_EXPECT_OK(parallel_graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(parallel_graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size(), output_packets_.size());
+  const TimedBoxProtoList parallel_tracking_result =
+      output_packets_[end_frame].Get<TimedBoxProtoList>();
+  EXPECT_EQ(1, parallel_tracking_result.box_size());
+
+  // should have only 1 random access
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const TimedBoxProtoList& parallel_ra_result =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  EXPECT_EQ(2, parallel_ra_result.box_size());
+
+  // Check if randan access tracking has same result with normal tracking.
+  ExpectQuadNear(parallel_tracking_result.box(0), parallel_ra_result.box(0));
+  ExpectQuadAtFrame(parallel_ra_result.box(0), end_frame - start_frame,
+                    kImageAspectRatio, false);
+  ExpectQuadAtFrame(parallel_ra_result.box(1),
+                    start_frame - reverse_start_frame - 1, kImageAspectRatio,
+                    false);
+}
+
+// Tests what happens when random access request timestamps are
+// outside of cache.
+TEST_F(TrackingGraphTest, TestRandomAccessTrackingTimestamps) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  // We intentionally don't cache all frames, to see what happens when
+  // random access tracking request time falls outside cache range.
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->set_streaming_track_data_cache_size(input_frames_packets_.size() - 1);
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  // Set up random access boxes
+  const int num_frames = input_frames_packets_.size();
+  const int64 usec_in_sec = 1000000;
+  std::vector<Timestamp> start_timestamps{
+      input_frames_packets_[0].Timestamp() - usec_in_sec,  // forward
+      input_frames_packets_[0].Timestamp(),                // forward
+      input_frames_packets_[1].Timestamp(),                // forward
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec,  // fwd
+      input_frames_packets_[0].Timestamp(),               // backward
+      input_frames_packets_[num_frames - 1].Timestamp(),  // backward
+      input_frames_packets_[num_frames - 1].Timestamp(),  // backward
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec  // back
+  };
+  std::vector<Timestamp> end_timestamps{
+      input_frames_packets_[num_frames - 1].Timestamp(),
+      input_frames_packets_[num_frames - 1].Timestamp(),
+      input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec,
+      input_frames_packets_[num_frames - 1].Timestamp() + 2 * usec_in_sec,
+      input_frames_packets_[0].Timestamp() - usec_in_sec,
+      input_frames_packets_[0].Timestamp(),
+      input_frames_packets_[0].Timestamp() - usec_in_sec,
+      input_frames_packets_[1].Timestamp()};
+  auto ra_boxes =
+      CreateRandomAccessTrackingBoxList(start_timestamps, end_timestamps);
+  Packet ra_packet =
+      Adopt(ra_boxes.release()).At(input_frames_packets_[0].Timestamp());
+
+  // Run the graph and check if the outside-cache request have no results.
+  // Start running the parallel graph, verify random access produce same result
+  // as normal tracking.
+  MP_EXPECT_OK(parallel_graph_.StartRun(side_packets));
+  for (auto frame_packet : input_frames_packets_) {
+    MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames",
+                                                        frame_packet));
+    MP_EXPECT_OK(parallel_graph_.WaitUntilIdle());
+  }
+  MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet));
+  MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(parallel_graph_.WaitUntilDone());
+
+  // should have 1 random access packet with 0 result boxes
+  EXPECT_EQ(1, random_access_results_packets_.size());
+  const auto& ra_returned_boxes =
+      random_access_results_packets_[0].Get<TimedBoxProtoList>();
+  const int num_returned_ra_boxes = ra_returned_boxes.box_size();
+  EXPECT_EQ(0, num_returned_ra_boxes);
+}
+
+TEST_F(TrackingGraphTest, TestTransitionFramesForReacquisition) {
+  // Create input side packets.
+  std::map<std::string, mediapipe::Packet> side_packets;
+  side_packets.insert(std::make_pair("analysis_downsample_factor",
+                                     mediapipe::MakePacket<float>(1.0f)));
+  CalculatorOptions calculator_options;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->mutable_tracker_options()
+      ->mutable_track_step_options()
+      ->set_tracking_degrees(
+          TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE);
+  constexpr int kTransitionFrames = 3;
+  calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext)
+      ->set_start_pos_transition_frames(kTransitionFrames);
+
+  side_packets.insert(std::make_pair(
+      "calculator_options",
+      mediapipe::MakePacket<CalculatorOptions>(calculator_options)));
+
+  Timestamp start_box_time = input_frames_packets_[0].Timestamp();
+  // Box id 0 use quad tracking with 8DoF homography transform.
+  // Box id 1 use quad tracking with 6DoF perspective transform.
+  // Box id 2 use box tracking with 4DoF similarity transform.
+  std::vector<bool> is_quad_tracking{true, true, false};
+  std::vector<bool> is_pnp_tracking{false, true, false};
+  std::vector<bool> is_reacquisition{true, true, true};
+  auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking,
+                                    is_pnp_tracking, is_reacquisition);
+  Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time);
+
+  // Setting box pos restart from initial position (frame 0's position).
+  constexpr int kRestartFrame = 3;
+  Timestamp restart_box_time = input_frames_packets_[kRestartFrame].Timestamp();
+  auto restart_box_list = MakeBoxList(restart_box_time, is_quad_tracking,
+                                      is_pnp_tracking, is_reacquisition);
+  Packet restart_pos_packet =
+      Adopt(restart_box_list.release()).At(restart_box_time);
+  MP_EXPECT_OK(graph_.StartRun(side_packets));
+  MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet));
+
+  for (int j = 0; j < input_frames_packets_.size(); ++j) {
+    // Add TRACK_TIME stream queries in between 2 frames.
+    if (j > 0) {
+      Timestamp track_time = Timestamp((j - 0.5f) * kFrameIntervalUs);
+      LOG(INFO) << track_time.Value();
+      Packet track_time_packet = Adopt(new Timestamp).At(track_time);
+      MP_EXPECT_OK(
+          graph_.AddPacketToInputStream("track_time", track_time_packet));
+    }
+
+    MP_EXPECT_OK(graph_.AddPacketToInputStream("image_cpu_frames",
+                                               input_frames_packets_[j]));
+    Packet track_time_packet =
+        Adopt(new int(0)).At(input_frames_packets_[j].Timestamp());
+    MP_EXPECT_OK(
+        graph_.AddPacketToInputStream("track_time", track_time_packet));
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+
+    if (j == kRestartFrame) {
+      MP_EXPECT_OK(
+          graph_.AddPacketToInputStream("restart_pos", restart_pos_packet));
+    }
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+
+  EXPECT_EQ(input_frames_packets_.size() * 2 - 1, output_packets_.size());
+  for (int i = 0; i < output_packets_.size(); ++i) {
+    const TimedBoxProtoList& boxes =
+        output_packets_[i].Get<TimedBoxProtoList>();
+    EXPECT_EQ(is_quad_tracking.size(), boxes.box_size());
+    float frame_id = i / 2.0f;
+    float expected_frame_id;
+    if (frame_id <= kRestartFrame) {
+      // before transition
+      expected_frame_id = frame_id;
+    } else {
+      float transition_frames = frame_id - kRestartFrame;
+      if (transition_frames <= kTransitionFrames) {
+        // transitioning.
+        expected_frame_id =
+            kRestartFrame -
+            transition_frames / kTransitionFrames * kRestartFrame +
+            transition_frames;
+      } else {
+        // after transition.
+        expected_frame_id = transition_frames;
+      }
+    }
+
+    for (int j = 0; j < boxes.box_size(); ++j) {
+      const TimedBoxProto& box = boxes.box(j);
+      if (is_quad_tracking[box.id()]) {
+        ExpectQuadAtFrame(box, expected_frame_id,
+                          is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f,
+                          is_reacquisition[box.id()]);
+      } else {
+        ExpectBoxAtFrame(box, expected_frame_id, is_reacquisition[box.id()]);
+      }
+    }
+  }
+}
+
+// TODO: Add test for reacquisition.
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/video/video_pre_stream_calculator.cc

@@ -0,0 +1,142 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/video/video_pre_stream_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+
+namespace mediapipe {
+
+// Sets up VideoHeader based on the 1st ImageFrame and emits it with timestamp
+// PreStream. Note that this calculator only fills in format, width, and height,
+// i.e. frame_rate and duration will not be filled, unless:
+// 1) an existing VideoHeader is provided at PreStream(). In such case, the
+//    frame_rate and duration, if they exist, will be copied from the existing
+//    VideoHeader.
+// 2) you specify frame_rate and duration through the options. In this case, the
+//    options will overwrite the existing VideoHeader if it is available.
+//
+// Example config:
+// node {
+//   calculator: "VideoPreStreamCalculator"
+//   input_stream: "FRAME:cropped_frames"
+//   input_stream: "VIDEO_PRESTREAM:original_video_header"
+//   output_stream: "cropped_frames_video_header"
+// }
+//
+// or
+//
+// node {
+//   calculator: "VideoPreStreamCalculator"
+//   input_stream: "cropped_frames"
+//   output_stream: "video_header"
+// }
+class VideoPreStreamCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::Status ProcessWithFrameRateInPreStream(CalculatorContext* cc);
+  ::mediapipe::Status ProcessWithFrameRateInOptions(CalculatorContext* cc);
+
+  std::unique_ptr<VideoHeader> header_;
+  bool frame_rate_in_prestream_ = false;
+  bool emitted_ = false;
+};
+
+REGISTER_CALCULATOR(VideoPreStreamCalculator);
+
+::mediapipe::Status VideoPreStreamCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (!cc->Inputs().UsesTags()) {
+    cc->Inputs().Index(0).Set<ImageFrame>();
+  } else {
+    cc->Inputs().Tag("FRAME").Set<ImageFrame>();
+    cc->Inputs().Tag("VIDEO_PRESTREAM").Set<VideoHeader>();
+  }
+  cc->Outputs().Index(0).Set<VideoHeader>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoPreStreamCalculator::Open(CalculatorContext* cc) {
+  frame_rate_in_prestream_ = cc->Inputs().UsesTags() &&
+                             cc->Inputs().HasTag("FRAME") &&
+                             cc->Inputs().HasTag("VIDEO_PRESTREAM");
+  header_ = absl::make_unique<VideoHeader>();
+  return ::mediapipe::OkStatus();
+}
+::mediapipe::Status VideoPreStreamCalculator::ProcessWithFrameRateInPreStream(
+    CalculatorContext* cc) {
+  cc->GetCounter("ProcessWithFrameRateInPreStream")->Increment();
+  if (cc->InputTimestamp() == Timestamp::PreStream()) {
+    RET_CHECK(cc->Inputs().Tag("FRAME").IsEmpty());
+    RET_CHECK(!cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty());
+    *header_ = cc->Inputs().Tag("VIDEO_PRESTREAM").Get<VideoHeader>();
+    RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+  } else {
+    RET_CHECK(cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty())
+        << "Packet on VIDEO_PRESTREAM must come in at Timestamp::PreStream().";
+    RET_CHECK(!cc->Inputs().Tag("FRAME").IsEmpty());
+    const auto& frame = cc->Inputs().Tag("FRAME").Get<ImageFrame>();
+    header_->format = frame.Format();
+    header_->width = frame.Width();
+    header_->height = frame.Height();
+    RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+    cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream());
+    emitted_ = true;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoPreStreamCalculator::Process(CalculatorContext* cc) {
+  cc->GetCounter("Process")->Increment();
+  if (emitted_) {
+    return ::mediapipe::OkStatus();
+  }
+  if (frame_rate_in_prestream_) {
+    return ProcessWithFrameRateInPreStream(cc);
+  } else {
+    return ProcessWithFrameRateInOptions(cc);
+  }
+}
+
+::mediapipe::Status VideoPreStreamCalculator::ProcessWithFrameRateInOptions(
+    CalculatorContext* cc) {
+  cc->GetCounter("ProcessWithFrameRateInOptions")->Increment();
+  RET_CHECK_NE(cc->InputTimestamp(), Timestamp::PreStream());
+  const auto& frame = cc->Inputs().Index(0).Get<ImageFrame>();
+  header_->format = frame.Format();
+  header_->width = frame.Width();
+  header_->height = frame.Height();
+  const auto& options = cc->Options<VideoPreStreamCalculatorOptions>();
+  if (options.fps().has_value()) {
+    header_->frame_rate = options.fps().value();
+  } else if (options.fps().has_ratio()) {
+    const VideoPreStreamCalculatorOptions::Fps::Rational32& ratio =
+        options.fps().ratio();
+    if (ratio.numerator() > 0 && ratio.denominator() > 0) {
+      header_->frame_rate =
+          static_cast<double>(ratio.numerator()) / ratio.denominator();
+    }
+  }
+  RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+  cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream());
+  emitted_ = true;
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/video_pre_stream_calculator.proto

@@ -0,0 +1,43 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message VideoPreStreamCalculatorOptions {
+  extend CalculatorOptions {
+    optional VideoPreStreamCalculatorOptions ext = 151386123;
+  }
+
+  // An arbitrary number of frames per second.
+  // Prefer the StandardFps enum to store industry-standard, safe FPS values.
+  message Fps {
+    // The possibly approximated value of the frame rate, in frames per second.
+    // Unsafe to use in accurate computations because prone to rounding errors.
+    // For example, the 23.976 FPS value has no exact representation as a
+    // double.
+    optional double value = 1;
+
+    message Rational32 {
+      optional int32 numerator = 1;
+      optional int32 denominator = 2;
+    }
+    // The exact value of the frame rate, as a rational number.
+    optional Rational32 ratio = 2;
+  }
+  optional Fps fps = 1;
+}

mediapipe/calculators/video/video_pre_stream_calculator_test.cc

@@ -0,0 +1,186 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "input"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "input"
+      output_stream: "output"
+      options {
+        [mediapipe.VideoPreStreamCalculatorOptions.ext] { fps { value: 3 } }
+      }
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto poller_status = graph.AddOutputStreamPoller("output");
+  MP_ASSERT_OK(poller_status.status());
+  OutputStreamPoller& poller = poller_status.ValueOrDie();
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+
+  // It is *not* VideoPreStreamCalculator's job to detect errors in an
+  // ImageFrame stream.  It just waits for the 1st ImageFrame, extracts info for
+  // VideoHeader, and emits it.  Thus, the following is fine.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input",
+      Adopt(new ImageFrame(ImageFormat::SRGBA, 3, 4)).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  const auto& video_header = packet.Get<VideoHeader>();
+  EXPECT_EQ(video_header.format, ImageFormat::SRGB);
+  EXPECT_EQ(video_header.width, 1);
+  EXPECT_EQ(video_header.height, 2);
+  EXPECT_EQ(video_header.frame_rate, 3);
+  EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream());
+  ASSERT_FALSE(poller.Next(&packet));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+}
+
+TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto poller_status = graph.AddOutputStreamPoller("output_header");
+  MP_ASSERT_OK(poller_status.status());
+  OutputStreamPoller& poller = poller_status.ValueOrDie();
+  MP_ASSERT_OK(graph.StartRun({}));
+  auto input_header = absl::make_unique<VideoHeader>();
+  input_header->frame_rate = 3.0;
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_header",
+      Adopt(input_header.release()).At(Timestamp::PreStream())));
+  MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  const auto& output_header = packet.Get<VideoHeader>();
+  EXPECT_EQ(output_header.format, ImageFormat::SRGB);
+  EXPECT_EQ(output_header.width, 1);
+  EXPECT_EQ(output_header.height, 2);
+  EXPECT_EQ(output_header.frame_rate, 3.0);
+  EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream());
+  ASSERT_FALSE(poller.Next(&packet));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+}
+
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInOptions) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "frame"
+      output_stream: "output_header"
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  ::mediapipe::Status status = graph.WaitUntilDone();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr("frame rate should be non-zero"));
+}
+
+// Input header missing.
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    }
+  )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+  ::mediapipe::Status status = graph.WaitUntilDone();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr("frame rate should be non-zero"));
+}
+
+// Input header not at prestream (before, with, and after frame data).
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream2) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    }
+  )");
+
+  for (int64 timestamp = -1; timestamp < 2; ++timestamp) {
+    CalculatorGraph graph;
+    MP_ASSERT_OK(graph.Initialize(config));
+    MP_ASSERT_OK(graph.StartRun({}));
+    auto input_header = absl::make_unique<VideoHeader>();
+    input_header->frame_rate = 3.0;
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input_header",
+        Adopt(input_header.release()).At(Timestamp(timestamp))));
+    MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "frame",
+        Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+    MP_ASSERT_OK(graph.CloseInputStream("frame"));
+    ::mediapipe::Status status = graph.WaitUntilDone();
+    EXPECT_FALSE(status.ok());
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/docs/autoflip.md

@@ -0,0 +1,321 @@
+# Saliency-Aware Video Cropping using AutoFlip
+
+## Introduction
+
+AutoFlip is an automatic video cropping pipeline built on top of MediaPipe. This
+example focuses on demonstrating how to use AutoFlip to convert an input video
+to arbitrary aspect ratios.
+
+For overall context on AutoFlip, please read this
+[Google AI Blog](https://mediapipe.page.link/autoflip).
+
+![graph is_required](images/autoflip_edited_example.gif)
+
+## Building
+
+Run the following command to build the AutoFlip pipeline:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/autoflip:run_autoflip
+```
+
+## Running
+
+```bash
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
+  --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
+  --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
+    output_video_path=/absolute/path/to/save/the/output/video/file,\
+    aspect_ratio=9:16
+```
+
+Use the `aspect_ratio` flag to provide the output aspect ratio. The format
+should be `weight:height`, where the `weight` and `height` are two positive
+integers. AutoFlip supports both landscape-to-portrait and portrait-to-landscape
+conversions. The pipeline internally compares the target aspect ratio against
+the original one, and determines the correct conversion automatically.
+
+We have put a couple test videos under this
+[Google Drive folder](https://drive.google.com/corp/drive/u/0/folders/1KK9LV--Ey0UEVpxssVLhVl7dypgJSQgk).
+You could download the videos into your local file system, then modify the
+command above accordingly to run AutoFlip against the videos.
+
+## MediaPipe Graph
+
+![graph visualization](images/autoflip_graph.png)
+
+To visualize the graph as shown above, copy the text specification of the graph
+below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev).
+
+```bash
+# Autoflip graph that only renders the final cropped video. For use with
+# end user applications.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  node_options: {
+    [type.googleapis.com/mediapipe.ScaleImageCalculatorOptions]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  node_options: {
+    [type.googleapis.com/mediapipe.PacketThinnerCalculatorOptions]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.ShotBoundaryCalculatorOptions] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.LocalizationToRegionCalculatorOptions] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.SignalFusingCalculatorOptions] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  node_options: {
+    [type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+```
+
+## Advanced Parameters
+
+### Required vs. Best-Effort Saliency Features
+
+AutoFlip allows users to implement and specify custom features to be used in the
+camera trajectory computation. If the user would like to detect and preserve
+scenes of lions in a wildlife protection video, for example, they could
+implement and add a feature detection calculator for lions into the pipeline.
+Refer to `AutoFlipFaceDetectionSubgraph` and `FaceToRegionCalculator`, or
+`AutoFlipObjectDetectionSubgraph` and `LocalizationToRegionCalculator` for
+examples of how to create new feature detection calculators.
+
+After adding different feature signals into the graph, use the
+`SignalFusingCalculator` node to specify types and weights for different feature
+signals. For example, in the graph above, we specified a `face_region` and an
+`object_region` input streams, to represent face signals and agnostic object
+signals, respectively.
+
+The larger the weight, the more important the features will be considered when
+AutoFlip computes the camera trajectory. Use the `is_required` flag to mark a
+feature as a hard constraint, in which case the computed camera trajectory will
+try best to cover these feature types in the cropped videos. If for some reason
+the required features cannot be all covered (for example, when they are too
+spread out in the video), AutoFlip will apply a padding effect to cover as much
+salient content as possible. See an illustration below.
+
+![graph is_required](images/autoflip_is_required.gif)
+
+### Visualization to Facilitate Debugging
+
+`SceneCroppingCalculator` provides two extra output streams
+`KEY_FRAME_CROP_REGION_VIZ_FRAMES` and `SALIENT_POINT_FRAME_VIZ_FRAMES` to
+visualize the cropping window as well as salient points detected on each frame.
+You could modify the `SceneCroppingCalculator` node like below to enable these
+two output streams.
+
+```bash
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  node_options: {
+    [type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+```

mediapipe/docs/calculator.md

@@ -151,7 +151,6 @@ node {
    input_stream: "room_video_tick_signal"
    output_stream: "cloned_room_mic_signal"
    output_stream: "cloned_lighting_sensor"
-   output_stream: "cloned_video_tick_signal"
  }
 ```
 

mediapipe/docs/examples.md

@@ -39,6 +39,11 @@ pipeline.
 the same TFLite model in a CPU-based pipeline. This example highlights how
 graphs can be easily adapted to run on CPU v.s. GPU.
 
+### Object Detection and Tracking with GPU
+
+[Object Detection and Tracking with GPU](./object_tracking_mobile_gpu.md) illustrates how to
+use MediaPipe for object detection and tracking.
+
 ### Face Detection with GPU
 
 [Face Detection with GPU](./face_detection_mobile_gpu.md) illustrates how to use
@@ -103,9 +108,9 @@ and model details are described in the
 [Hello World for C++](./hello_world_desktop.md) shows how to run a simple graph
 using the MediaPipe C++ APIs.
 
-### Feature Extration for YouTube-8M Challenge
+### Feature Extraction and Model Inference for YouTube-8M Challenge
 
-[Feature Extration and Model Inference for YouTube-8M Challenge](./youtube_8m.md)
+[Feature Extraction and Model Inference for YouTube-8M Challenge](./youtube_8m.md)
 shows how to use MediaPipe to prepare training data for the YouTube-8M Challenge
 and do the model inference with the baseline model.
 
@@ -115,6 +120,11 @@ and do the model inference with the baseline model.
 MediaPipe for media processing to prepare video data sets for training a
 TensorFlow model.
 
+### Automatic video cropping
+
+[AutoFlip](./autoflip.md) shows how to use MediaPipe to build an automatic video
+cropping pipeline that can convert an input video to arbitrary aspect ratios.
+
 ### Object Detection on Desktop
 
 [Object Detection on Desktop](./object_detection_desktop.md) shows how to run

mediapipe/docs/face_detection_coral_devboard.md

@@ -2,9 +2,10 @@
 
 MediaPipe is able to run cross platform across device types like desktop, mobile
 and edge devices. Here is an example of running MediaPipe
-[face detection pipeline](./face_detection_desktop.md) on edge device like
+[face detection pipeline](./face_detection_desktop.md) on edge device like the
-[Google Coral dev board](https://coral.withgoogle.com/products/dev-board) with
+[Coral Dev Board](https://coral.ai/products/dev-board).
-[Edge TPU](https://cloud.google.com/edge-tpu/). This MediaPipe Coral face
+
+This MediaPipe Coral face
 detection pipeline is running [coral specific quantized version](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
 of the [MediaPipe face detection TFLite model](https://github.com/google/mediapipe/blob/master/mediapipe/models/face_detection_front.tflite)
 accelerated on Edge TPU.
@@ -13,8 +14,10 @@ accelerated on Edge TPU.
 
 We recommend building the MediaPipe binaries not on the edge device due to
 limited compute resulting in long build times. Instead, we will build MediaPipe
-binaries using Docker containers on a more powerful host machine. For step by
+binaries using Docker containers on a more powerful host machine.
-step details of cross compiling and running MediaPipe binaries on Coral dev
-board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/README.md).
 
-![Face Detection running on Coral](images/face_detection_demo_coral.jpg)
+For step by
+step details of cross compiling and running MediaPipe binaries on the Coral Dev
+Board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral).
+
+![Face Detection running on Coral](images/face_detection_demo_coral.gif)

mediapipe/docs/framework_concepts.md

@@ -24,9 +24,9 @@ Mediapipe. At a minimum, a new calculator must implement the below four methods
 * `GetContract()`
   * Calculator authors can specify the expected types of inputs and outputs of a calculator in GetContract(). When a graph is initialized, the framework calls a static method to verify if the packet types of the connected inputs and outputs match the information in this specification.
 * `Open()`
-  * After a graph starts, the framework calls `Open()`. The input side packets are available to the calculator at this point. `Open()` interprets the node configuration (see Section \ref{graph_config}) operations and prepares the calculator's per-graph-run state. This function may also write packets to calculator outputs. An error during `Open()` can terminate the graph run.
+  * After a graph starts, the framework calls `Open()`. The input side packets are available to the calculator at this point. `Open()` interprets the node configuration operations (see Section [GraphConfig](#graphconfig)) and prepares the calculator's per-graph-run state. This function may also write packets to calculator outputs. An error during `Open()` can terminate the graph run.
 * `Process()`
-  * For a calculator with inputs, the framework calls `Process()` repeatedly whenever at least one input stream has a packet available. The framework by default guarantees that all inputs have the same timestamp (see Section \ref{scheduling} for more information). Multiple `Process()` calls can be invoked simultaneously when parallel execution is enabled. If an error occurs during `Process()`, the framework calls `Close()` and the graph run terminates.
+  * For a calculator with inputs, the framework calls `Process()` repeatedly whenever at least one input stream has a packet available. The framework by default guarantees that all inputs have the same timestamp (see [Framework Architecture](scheduling_sync.md) for more information). Multiple `Process()` calls can be invoked simultaneously when parallel execution is enabled. If an error occurs during `Process()`, the framework calls `Close()` and the graph run terminates.
 * `Close()`
   * After all calls to `Process()` finish or when all input streams close, the framework calls `Close()`. This function is always called if `Open()` was called and succeeded and even if the graph run terminated because of an error. No inputs are available via any input streams during `Close()`, but it still has access to input side packets and therefore may write outputs. After `Close()` returns, the calculator should be considered a dead node. The calculator object is destroyed as soon as the graph finishes running.
 

mediapipe/docs/install.md

@@ -39,19 +39,12 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (0.24.1 and above required).
+2.  Install Bazel (version between 0.24.1 and 1.2.1).
 
-    Option 1. Use package manager tool to install the latest version of Bazel.
+    Follow the official
-
-    ```bash
-    $ sudo apt-get install bazel
-
-    # Run 'bazel version' to check version of bazel installed
-    ```
-
-    Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install any version of Bazel manually.
+    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    yet.
 
 3.  Install OpenCV and FFmpeg.
 
@@ -63,6 +56,7 @@ To build and run iOS apps:
 
     ```bash
     $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+                           libopencv-calib3d-dev libopencv-features2d-dev \
                            libopencv-imgproc-dev libopencv-video-dev
     ```
 
@@ -157,11 +151,12 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (0.24.1 and above required).
+2.  Install Bazel (version between 0.24.1 and 1.2.1).
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel manually.
+    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    yet.
 
 3.  Install OpenCV.
 
@@ -360,6 +355,9 @@ To build and run iOS apps:
 
 ### Installing on Windows Subsystem for Linux (WSL)
 
+Note: WSL has historically not provided access to USB cameras. Mediapipe can use
+a video file as input.
+
 1.  Follow the
     [instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
     install Windows Sysystem for Linux (Ubuntu).
@@ -379,10 +377,10 @@ To build and run iOS apps:
 4.  Install the needed packages.
 
     ```bash
-    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential git python zip adb openjdk-8-jdk
+    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel (0.24.1 and above required).
+5.  Install Bazel (version between 0.24.1 and 1.2.1).
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
@@ -411,6 +409,7 @@ To build and run iOS apps:
 
     ```bash
     username@DESKTOP-TMVLBJ1:~/mediapipe$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+                           libopencv-calib3d-dev libopencv-features2d-dev \
                            libopencv-imgproc-dev libopencv-video-dev
     ```
 

mediapipe/docs/object_detection_coral_devboard.md

@@ -3,8 +3,9 @@
 MediaPipe is able to run cross platform across device types like desktop, mobile
 and edge devices. Here is an example of running MediaPipe
 [object detection pipeline](./object_detection_desktop.md) on edge device like
-[Google Coral dev board](https://coral.withgoogle.com/products/dev-board) with
+the [Coral Dev Board](https://coral.ai/products/dev-board).
-[Edge TPU](https://cloud.google.com/edge-tpu/). This MediaPipe Coral object
+
+This MediaPipe Coral object
 detection pipeline is running [coral specific quantized version](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
 of the [MediaPipe object detection TFLite model](https://github.com/google/mediapipe/blob/master/mediapipe/models/object_detection_front.tflite)
 accelerated on Edge TPU.
@@ -13,8 +14,10 @@ accelerated on Edge TPU.
 
 We recommend building the MediaPipe binaries not on the edge device due to
 limited compute resulting in long build times. Instead, we will build MediaPipe
-binaries using Docker containers on a more powerful host machine. For step by
+binaries using Docker containers on a more powerful host machine.
-step details of cross compiling and running MediaPipe binaries on Coral dev
+
-board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/README.md).
+For step by
+step details of cross compiling and running MediaPipe binaries on the Coral Dev
+Board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral).
 
 ![Object Detection running on Coral](images/object_detection_demo_coral.jpg)

mediapipe/docs/object_tracking_mobile_gpu.md

@@ -0,0 +1,491 @@
+# Object Detection and Tracking
+
+This doc focuses on the
+[below example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/object_detection_tracking_mobile_gpu.pbtxt)
+that performs object detection and tracking.
+Note that object detection is using TensorFlow Lite on GPU while tracking is using CPU.
+
+For overall context on object detection and tracking, please read this
+[Google Developers Blog](https://mediapipe.page.link/objecttrackingblog).
+
+![object_tracking_android_gpu_gif](images/mobile/object_tracking_android_gpu.gif)
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu)
+
+To build and install the app:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/objecttrackinggpu.apk
+```
+
+## Graph
+The object detection and tracking [main graph](#main-graph) internally utilizes a
+[object detection subgraph](#object-detection-subgraph), a
+[object tracking subgraph](#object-tracking-subgraph) and a
+[renderer subgraph](#renderer-subgraph).
+
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+
+![object_detection_mobile_gpu_graph](images/mobile/object_tracking_mobile_gpu.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt)
+
+```bash
+# MediaPipe graph that performs object detection and tracking.
+# Used in the examples in
+# mediapipie/examples/android/src/java/com/mediapipe/apps/objecttrackinggpu
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Resamples the images by specific frame rate. This calculator is used to
+# control the frequecy of subsequent calculators/subgraphs, e.g. less power
+# consumption for expensive process.
+node {
+  calculator: "PacketResamplerCalculator"
+  input_stream: "DATA:input_video"
+  output_stream: "DATA:throttled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.PacketResamplerCalculatorOptions] {
+      frame_rate: 0.5
+    }
+  }
+}
+
+# Subgraph that detections objects (see object_detection_gpu.pbtxt).
+node {
+  calculator: "ObjectDetectionSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "DETECTIONS:output_detections"
+}
+
+# Subgraph that tracks objects (see object_tracking.pbtxt).
+node {
+  calculator: "ObjectTrackingSubgraph"
+  input_stream: "VIDEO:input_video"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "DETECTIONS:tracked_detections"
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see renderer_gpu.pbtxt).
+node {
+  calculator: "RendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "DETECTIONS:tracked_detections"
+  output_stream: "IMAGE:output_video"
+}
+```
+
+### Object Detection Subgraph
+
+![object_detection_gpu_subgraph](images/mobile/object_detection_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_detection_gpu.pbtxt)
+
+```bash
+# MediaPipe object detection subgraph.
+
+type: "ObjectDetectionSubgraph"
+
+input_stream: "IMAGE:input_video"
+output_stream: "DETECTIONS:output_detections"
+
+# Transforms the input image on GPU to a 320x320 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 320
+      output_height: 320
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS_GPU:detection_tensors"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/ssdlite_object_detection.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  node_options: {
+    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 320
+      input_size_width: 320
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
+      num_classes: 91
+      num_boxes: 2034
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 3
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt"
+    }
+  }
+}
+```
+
+### Object Tracking Subgraph
+
+Object tracking subgraph uses a box tracking subgraph which is a generic
+tracking library that can be used for other use cases.
+
+![object_tracking_subgraph](images/mobile/object_tracking_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_tracking_gpu.pbtxt)
+
+```bash
+# MediaPipe object tracking subgraph.
+
+type: "ObjectTrackingSubgraph"
+
+input_stream: "VIDEO:input_video"
+input_stream: "DETECTIONS:new_detections"
+output_stream: "DETECTIONS:tracked_detections"
+
+# Assigns an unique id for each new detection.
+node {
+  calculator: "DetectionUniqueIdCalculator"
+  input_stream: "DETECTIONS:new_detections"
+  output_stream: "DETECTIONS:detections_with_id"
+}
+
+# Converts detections to TimedBox protos which are used as initial location
+# for tracking.
+node {
+  calculator: "DetectionsToTimedBoxListCalculator"
+  input_stream: "DETECTIONS:detections_with_id"
+  output_stream: "BOXES:start_pos"
+}
+
+# Subgraph that tracks boxes (see box_tracking.pbtxt).
+node {
+  calculator: "BoxTrackingSubgraph"
+  input_stream: "VIDEO:input_video"
+  input_stream: "BOXES:start_pos"
+  input_stream: "CANCEL_ID:cancel_object_id"
+  output_stream: "BOXES:boxes"
+}
+
+# Managers new detected objects and objects that are being tracked.
+# It associates the duplicated detections and updates the locations of
+# detections from tracking.
+node: {
+  calculator: "TrackedDetectionManagerCalculator"
+  input_stream: "DETECTIONS:detections_with_id"
+  input_stream: "TRACKING_BOXES:boxes"
+  output_stream: "DETECTIONS:tracked_detections"
+  output_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "TRACKING_BOXES"
+        }
+        sync_set {
+          tag_index: "DETECTIONS"
+        }
+      }
+    }
+  }
+}
+
+```
+
+### Box Tracking Subgraph
+
+![box_tracking_subgraph](images/mobile/box_tracking_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking_gpu.pbtxt)
+
+```bash
+# MediaPipe box tracking subgraph.
+
+type: "BoxTrackingSubgraph"
+
+input_stream: "VIDEO:input_video"
+input_stream: "BOXES:start_pos"
+input_stream: "CANCEL_ID:cancel_object_id"
+output_stream: "BOXES:boxes"
+
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:downscaled_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 240
+      output_height: 320
+    }
+  }
+}
+
+# Converts GPU buffer to ImageFrame for processing tracking.
+node: {
+  calculator: "GpuBufferToImageFrameCalculator"
+  input_stream: "downscaled_input_video"
+  output_stream: "downscaled_input_video_cpu"
+}
+
+# Performs motion analysis on an incoming video stream.
+node: {
+  calculator: "MotionAnalysisCalculator"
+  input_stream: "VIDEO:downscaled_input_video_cpu"
+  output_stream: "CAMERA:camera_motion"
+  output_stream: "FLOW:region_flow"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.MotionAnalysisCalculatorOptions]: {
+      analysis_options {
+        analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE
+        flow_options {
+          fast_estimation_min_block_size: 100
+          top_inlier_sets: 1
+          frac_inlier_error_threshold: 3e-3
+          downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE
+          verification_distance: 5.0
+          verify_long_feature_acceleration: true
+          verify_long_feature_trigger_ratio: 0.1
+          tracking_options {
+            max_features: 500
+            adaptive_extraction_levels: 2
+            min_eig_val_settings {
+              adaptive_lowest_quality_level: 2e-4
+            }
+            klt_tracker_implementation: KLT_OPENCV
+          }
+        }
+      }
+    }
+  }
+}
+
+# Reads optical flow fields defined in
+# mediapipe/framework/formats/motion/optical_flow_field.h,
+# returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized
+# to 0-255.
+node: {
+  calculator: "FlowPackagerCalculator"
+  input_stream: "FLOW:region_flow"
+  input_stream: "CAMERA:camera_motion"
+  output_stream: "TRACKING:tracking_data"
+
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowPackagerCalculatorOptions]: {
+      flow_packager_options: {
+        binary_tracking_data_support: false
+      }
+    }
+  }
+}
+
+# Tracks box positions over time.
+node: {
+  calculator: "BoxTrackerCalculator"
+  input_stream: "TRACKING:tracking_data"
+  input_stream: "TRACK_TIME:input_video"
+  input_stream: "START_POS:start_pos"
+  input_stream: "CANCEL_OBJECT_ID:cancel_object_id"
+  input_stream_info: {
+    tag_index: "CANCEL_OBJECT_ID"
+    back_edge: true
+  }
+  output_stream: "BOXES:boxes"
+
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler"
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "TRACKING"
+          tag_index: "TRACK_TIME"
+        }
+        sync_set {
+          tag_index: "START_POS"
+        }
+        sync_set {
+          tag_index: "CANCEL_OBJECT_ID"
+        }
+      }
+    }
+  }
+
+  node_options: {
+    [type.googleapis.com/mediapipe.BoxTrackerCalculatorOptions]: {
+      tracker_options: {
+        track_step_options {
+          track_object_and_camera: true
+          tracking_degrees: TRACKING_DEGREE_OBJECT_SCALE
+          inlier_spring_force: 0.0
+          static_motion_temporal_ratio: 3e-2
+        }
+      }
+      visualize_tracking_data: false
+      streaming_track_data_cache_size: 100
+    }
+  }
+}
+```
+
+### Renderer Subgraph
+
+![object_tracking_renderer_gpu_subgraph.pbtxt](images/mobile/object_tracking_renderer_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt)
+
+```bash
+# MediaPipe object tracking rendering subgraph.
+
+type: "RendererSubgraph"
+
+input_stream: "IMAGE:input_image"
+input_stream: "DETECTIONS:detections"
+output_stream: "IMAGE:output_image"
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:detections"
+  output_stream: "RENDER_DATA:detections_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+      render_detection_id: true
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "detections_render_data"
+  output_stream: "OUTPUT_FRAME_GPU:output_image"
+}
+
+```
+
+## Desktop
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/object_tracking)
+
+Note that object detection is using TensorFlow Lite on CPU and tracking is using
+CPU.
+
+To build and run the app:
+
+```bash
+bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
+
+bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt
+```

mediapipe/docs/youtube_8m.md

@@ -1,4 +1,4 @@
-# Feature Extration and Model Inference for YouTube-8M Challenge
+# Feature Extraction and Model Inference for YouTube-8M Challenge
 
 MediaPipe is a useful and general framework for media processing that can assist
 with research, development, and deployment of ML models. This example focuses on
@@ -31,7 +31,9 @@ videos.
 
 ### Steps to run the YouTube-8M feature extraction graph
 
-1.  Checkout the mediapipe repository.
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
 
     ```bash
     git clone https://github.com/google/mediapipe.git

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD

@@ -61,7 +61,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD

@@ -55,6 +55,7 @@ android_library(
     resource_files = glob(["res/**"]),
     deps = [
         ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:detection_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
         "//mediapipe/java/com/google/mediapipe/components:android_components",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",
@@ -62,7 +63,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java

@@ -17,18 +17,22 @@ package com.google.mediapipe.apps.objectdetectiongpu;
 import android.graphics.SurfaceTexture;
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
 import com.google.mediapipe.components.CameraHelper;
 import com.google.mediapipe.components.CameraXPreviewHelper;
 import com.google.mediapipe.components.ExternalTextureConverter;
 import com.google.mediapipe.components.FrameProcessor;
 import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.glutil.EglManager;
+import java.util.List;
 
 /** Main activity of MediaPipe example apps. */
 public class MainActivity extends AppCompatActivity {
@@ -37,6 +41,7 @@ public class MainActivity extends AppCompatActivity {
   private static final String BINARY_GRAPH_NAME = "objectdetectiongpu.binarypb";
   private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
   private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_DETECTIONS_STREAM_NAME = "output_detections";
   private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
 
   // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
@@ -90,6 +95,14 @@ public class MainActivity extends AppCompatActivity {
             OUTPUT_VIDEO_STREAM_NAME);
     processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
 
+    processor.addPacketCallback(
+        OUTPUT_DETECTIONS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received detections packet.");
+          List<Detection> detections = PacketGetter.getProtoVector(packet, Detection.parser());
+          Log.d(TAG, "[TS:" + packet.getTimestamp() + "] " + getDetectionsDebugString(detections));
+        });
+
     PermissionHelper.checkAndRequestCameraPermissions(this);
   }
 
@@ -164,4 +177,22 @@ public class MainActivity extends AppCompatActivity {
         });
     cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
   }
+
+  private static String getDetectionsDebugString(List<Detection> detections) {
+    if (detections.isEmpty()) {
+      return "No detections";
+    }
+    String detectionsStr = "Number of objects detected: " + detections.size() + "\n";
+    int objectIndex = 0;
+    for (Detection detection : detections) {
+      detectionsStr += "\t#Object[" + objectIndex + "]: \n";
+      List<String> labels = detection.getLabelList();
+      List<Float> scores = detection.getScoreList();
+      for (int i = 0; i < labels.size(); ++i) {
+        detectionsStr += "\t\tLabel [" + i + "]: " + labels.get(i) + ", " + scores.get(i) + "\n";
+      }
+      ++objectIndex;
+    }
+    return detectionsStr;
+  }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/AndroidManifest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.objecttrackinggpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD

@@ -0,0 +1,81 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/tracking:mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/tracking:mobile_gpu_binary_graph"],
+    outs = ["objecttrackinggpu.binarypb"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@androidx_concurrent_futures//jar",
+        "@androidx_lifecycle//jar",
+        "@com_google_guava_android//jar",
+    ],
+)
+
+android_binary(
+    name = "objecttrackinggpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.objecttrackinggpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/MainActivity.java

@@ -0,0 +1,167 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.objecttrackinggpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.glutil.EglManager;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "objecttrackinggpu.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java3");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/layout/activity_main.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Object Tracking GPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/res/values/styles.xml

@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>

mediapipe/examples/coral/README.md

@@ -1,8 +1,12 @@
 # Coral Dev Board Setup (experimental)
 
-**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may not be exact and is subject to change. These instructions have only been tested on the coral dev board with OS version _mendel day_, and may vary for different devices and workstations.
+**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may
+not be exact and is subject to change. These instructions have only been tested
+on the [Coral Dev Board](https://coral.ai/products/dev-board/) with Mendel 4.0,
+and may vary for different devices and workstations.
 
-This file describes how to prepare a Google Coral Dev Board and setup a linux Docker container for building MediaPipe applications that run on Edge TPU.
+This file describes how to prepare a Coral Dev Board and setup a Linux
+Docker container for building MediaPipe applications that run on Edge TPU.
 
 ## Before creating the Docker
 

mediapipe/examples/coral/demo_run_graph_main.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 //
 // An example of sending OpenCV webcam frames into a MediaPipe graph.
+#include <cstdlib>
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -89,7 +90,6 @@ DEFINE_string(output_video_path, "",
   MP_RETURN_IF_ERROR(graph.StartRun({}));
 
   LOG(INFO) << "Start grabbing and processing frames.";
-  size_t frame_timestamp = 0;
   bool grab_frames = true;
   while (grab_frames) {
     // Capture opencv camera or video frame.
@@ -110,9 +110,11 @@ DEFINE_string(output_video_path, "",
     camera_frame.copyTo(input_frame_mat);
 
     // Send image packet into the graph.
+    size_t frame_timestamp_us =
+        (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6;
     MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
         kInputStream, mediapipe::Adopt(input_frame.release())
-                          .At(mediapipe::Timestamp(frame_timestamp++))));
+                          .At(mediapipe::Timestamp(frame_timestamp_us))));
 
     // Get the graph result packet, or stop if that fails.
     mediapipe::Packet packet;
@@ -144,8 +146,9 @@ int main(int argc, char** argv) {
   ::mediapipe::Status run_status = RunMPPGraph();
   if (!run_status.ok()) {
     LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+    return EXIT_FAILURE;
   } else {
     LOG(INFO) << "Success!";
   }
-  return 0;
+  return EXIT_SUCCESS;
 }

mediapipe/examples/desktop/autoflip/BUILD

@@ -0,0 +1,56 @@
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+proto_library(
+    name = "autoflip_messages_proto",
+    srcs = ["autoflip_messages.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "autoflip_messages_cc_proto",
+    srcs = ["autoflip_messages.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe/examples:__subpackages__"],
+    deps = [":autoflip_messages_proto"],
+)
+
+cc_binary(
+    name = "run_autoflip",
+    deps = [
+        "//mediapipe/calculators/core:packet_thinner_calculator",
+        "//mediapipe/calculators/image:scale_image_calculator",
+        "//mediapipe/calculators/video:opencv_video_decoder_calculator",
+        "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+        "//mediapipe/calculators/video:video_pre_stream_calculator",
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/examples/desktop/autoflip/calculators:border_detection_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:face_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:localization_to_region_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:scene_cropping_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:shot_boundary_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:signal_fusing_calculator",
+        "//mediapipe/examples/desktop/autoflip/calculators:video_filtering_calculator",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_face_detection_subgraph",
+        "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_object_detection_subgraph",
+    ],
+)

mediapipe/examples/desktop/autoflip/README.md

@@ -0,0 +1,25 @@
+### Steps to run the AutoFlip video cropping graph
+
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
+
+    ```bash
+    git clone https://github.com/google/mediapipe.git
+    cd mediapipe
+    ```
+
+2.  Build and run the run_autoflip binary to process a local video.
+
+    ```bash
+    bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+      mediapipe/examples/desktop/autoflip:run_autoflip
+
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
+      --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
+      --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,\
+        output_video_path=/absolute/path/to/save/the/output/video/file,\
+        aspect_ratio=width:height
+    ```
+
+3.  View the cropped video.

mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt

@@ -0,0 +1,202 @@
+# Autoflip graph that only renders the final cropped video. For use with
+# end user applications.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  options: {
+    [mediapipe.ScaleImageCalculatorOptions.ext]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  options: {
+    [mediapipe.PacketThinnerCalculatorOptions.ext]: {
+      thinner_type: ASYNC
+      period: 500000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  options: {
+    [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.3
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  options: {
+    [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}