Project import generated by Copybara.

GitOrigin-RevId: 852dfb05d450167899c0dd5ef7c45622a12e865b

.bazelrc

@@ -12,7 +12,7 @@ build --copt='-Wno-comment'
 build --copt='-Wno-return-type'
 build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
-# Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
+# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
 build --incompatible_disable_deprecated_attr_params=false
 build --incompatible_depset_is_not_iterable=false
 

README.md

@@ -1,7 +1,7 @@
 ![MediaPipe](mediapipe/docs/images/mediapipe_small.png?raw=true "MediaPipe logo")
 =======================================================================
 
-[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
+[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data), cross platform (i.e Android, iOS, web, edge devices) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
 
 ![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)
 
@@ -9,16 +9,17 @@
 
 ## ML Solutions in MediaPipe
 
-* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
 * [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
-* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
-* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 * [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
+* [AutoFlip](mediapipe/docs/autoflip.md)
 
-![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
-![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
 ![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
 
@@ -28,6 +29,8 @@ Follow these [instructions](mediapipe/docs/install.md).
 ## Getting started
 See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
 
+Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+
 ## Documentation
 [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
 
@@ -36,10 +39,12 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 ## Visualizing MediaPipe graphs
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
-## Community forum
-*  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
+## Videos
+*  [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
 
 ## Publications
+* [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
+* [Google Developer Blog: Object Detection and Tracking using MediaPipe](https://mediapipe.page.link/objecttrackingblog)
 * [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
@@ -54,6 +59,9 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
 * [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
 
+## Community forum
+*  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
+
 ## Alpha Disclaimer
 MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
 

WORKSPACE

@@ -10,7 +10,9 @@ http_archive(
     sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.24.1")
+versions.check(minimum_bazel_version = "0.24.1",
+               maximum_bazel_version = "1.2.1")
+
 
 # ABSL cpp library lts_2019_08_08.
 http_archive(
@@ -76,6 +78,14 @@ http_archive(
     ],
 )
 
+# easyexif
+http_archive(
+    name = "easyexif",
+    url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
+    strip_prefix = "easyexif-master",
+    build_file = "@//third_party:easyexif.BUILD",
+)
+
 # libyuv
 http_archive(
     name = "libyuv",

mediapipe/calculators/core/BUILD

@@ -47,6 +47,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "packet_thinner_calculator_proto",
+    srcs = ["packet_thinner_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "split_vector_calculator_proto",
     srcs = ["split_vector_calculator.proto"],
@@ -79,6 +86,15 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "constant_side_packet_calculator_proto",
+    srcs = ["constant_side_packet_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 proto_library(
     name = "clip_vector_size_calculator_proto",
     srcs = ["clip_vector_size_calculator.proto"],
@@ -102,6 +118,14 @@ mediapipe_cc_proto_library(
     deps = [":packet_resampler_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "packet_thinner_calculator_cc_proto",
+    srcs = ["packet_thinner_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":packet_thinner_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "split_vector_calculator_cc_proto",
     srcs = ["split_vector_calculator.proto"],
@@ -158,6 +182,14 @@ mediapipe_cc_proto_library(
     deps = [":gate_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "constant_side_packet_calculator_cc_proto",
+    srcs = ["constant_side_packet_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":constant_side_packet_calculator_proto"],
+)
+
 cc_library(
     name = "add_header_calculator",
     srcs = ["add_header_calculator.cc"],
@@ -284,7 +316,6 @@ cc_test(
     srcs = ["concatenate_vector_calculator_test.cc"],
     deps = [
         ":concatenate_vector_calculator",
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:timestamp",
@@ -451,6 +482,37 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "packet_thinner_calculator",
+    srcs = ["packet_thinner_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "packet_thinner_calculator_test",
+    srcs = ["packet_thinner_calculator_test.cc"],
+    deps = [
+        ":packet_thinner_calculator",
+        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "pass_through_calculator",
     srcs = ["pass_through_calculator.cc"],
@@ -572,6 +634,7 @@ cc_test(
 cc_library(
     name = "packet_resampler_calculator",
     srcs = ["packet_resampler_calculator.cc"],
+    hdrs = ["packet_resampler_calculator.h"],
     visibility = [
         "//visibility:public",
     ],
@@ -595,17 +658,17 @@ cc_library(
 cc_test(
     name = "packet_resampler_calculator_test",
     timeout = "short",
-    srcs = ["packet_resampler_calculator_test.cc"],
+    srcs = [
+        "packet_resampler_calculator_test.cc",
+    ],
     deps = [
         ":packet_resampler_calculator",
         "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:video_stream_header",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
-        "//mediapipe/framework/port:status",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -698,7 +761,13 @@ cc_library(
         "//mediapipe/util:resource_util",
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
-    ],
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+        ],
+    }),
     alwayslink = 1,
 )
 
@@ -908,3 +977,30 @@ cc_test(
         "@com_google_absl//absl/memory",
     ],
 )
+
+cc_library(
+    name = "constant_side_packet_calculator",
+    srcs = ["constant_side_packet_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":constant_side_packet_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "constant_side_packet_calculator_test",
+    srcs = ["constant_side_packet_calculator_test.cc"],
+    deps = [
+        ":constant_side_packet_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/calculators/core/clip_vector_size_calculator.h

@@ -38,6 +38,8 @@ namespace mediapipe {
 //     }
 //   }
 // }
+// Optionally, you can pass in a side packet that will override `max_vec_size`
+// that is specified in the options.
 template <typename T>
 class ClipVectorSizeCalculator : public CalculatorBase {
  public:
@@ -53,6 +55,10 @@ class ClipVectorSizeCalculator : public CalculatorBase {
 
     cc->Inputs().Index(0).Set<std::vector<T>>();
     cc->Outputs().Index(0).Set<std::vector<T>>();
+    // Optional input side packet that determines `max_vec_size`.
+    if (cc->InputSidePackets().NumEntries() > 0) {
+      cc->InputSidePackets().Index(0).Set<int>();
+    }
 
     return ::mediapipe::OkStatus();
   }
@@ -61,6 +67,11 @@ class ClipVectorSizeCalculator : public CalculatorBase {
     cc->SetOffset(TimestampDiff(0));
     max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
                         .max_vec_size();
+    // Override `max_vec_size` if passed as side packet.
+    if (cc->InputSidePackets().NumEntries() > 0 &&
+        !cc->InputSidePackets().Index(0).IsEmpty()) {
+      max_vec_size_ = cc->InputSidePackets().Index(0).Get<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -176,4 +176,31 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
   }
 }
 
+TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        input_side_packet: "max_vec_size"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  // This should override the default of 1 set in the options.
+  runner.MutableSidePackets()->Index(0) = Adopt(new int(2));
+  std::vector<int> input = {0, 1, 2, 3};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+  EXPECT_EQ(2, output.size());
+  std::vector<int> expected_vector = {0, 1};
+  EXPECT_EQ(expected_vector, output);
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -35,6 +35,16 @@ namespace mediapipe {
 typedef ConcatenateVectorCalculator<float> ConcatenateFloatVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
 
+// Example config:
+// node {
+//   calculator: "ConcatenateInt32VectorCalculator"
+//   input_stream: "int32_vector_1"
+//   input_stream: "int32_vector_2"
+//   output_stream: "concatenated_int32_vector"
+// }
+typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
+REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -0,0 +1,116 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Generates an output side packet or multiple output side packets according to
+// the specified options.
+//
+// Example configs:
+// node {
+//   calculator: "ConstantSidePacketCalculator"
+//   output_side_packet: "PACKET:packet"
+//   options: {
+//     [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+//       packet { int_value: 2 }
+//     }
+//   }
+// }
+//
+// node {
+//   calculator: "ConstantSidePacketCalculator"
+//   output_side_packet: "PACKET:0:int_packet"
+//   output_side_packet: "PACKET:1:bool_packet"
+//   options: {
+//     [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+//       packet { int_value: 2 }
+//       packet { bool_value: true }
+//     }
+//   }
+// }
+class ConstantSidePacketCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    const auto& options = cc->Options().GetExtension(
+        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
+                 options.packet_size())
+        << "Number of output side packets has to be same as number of packets "
+           "configured in options.";
+
+    int index = 0;
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
+         id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
+      const auto& packet_options = options.packet(index);
+      auto& packet = cc->OutputSidePackets().Get(id);
+      if (packet_options.has_int_value()) {
+        packet.Set<int>();
+      } else if (packet_options.has_float_value()) {
+        packet.Set<float>();
+      } else if (packet_options.has_bool_value()) {
+        packet.Set<bool>();
+      } else if (packet_options.has_string_value()) {
+        packet.Set<std::string>();
+      } else {
+        return ::mediapipe::InvalidArgumentError(
+            "None of supported values were specified in options.");
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const auto& options = cc->Options().GetExtension(
+        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+    int index = 0;
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
+         id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {
+      auto& packet = cc->OutputSidePackets().Get(id);
+      const auto& packet_options = options.packet(index);
+      if (packet_options.has_int_value()) {
+        packet.Set(MakePacket<int>(packet_options.int_value()));
+      } else if (packet_options.has_float_value()) {
+        packet.Set(MakePacket<float>(packet_options.float_value()));
+      } else if (packet_options.has_bool_value()) {
+        packet.Set(MakePacket<bool>(packet_options.bool_value()));
+      } else if (packet_options.has_string_value()) {
+        packet.Set(MakePacket<std::string>(packet_options.string_value()));
+      } else {
+        return ::mediapipe::InvalidArgumentError(
+            "None of supported values were specified in options.");
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  static constexpr const char* kPacketTag = "PACKET";
+};
+
+REGISTER_CALCULATOR(ConstantSidePacketCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -0,0 +1,36 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ConstantSidePacketCalculatorOptions {
+  extend CalculatorOptions {
+    optional ConstantSidePacketCalculatorOptions ext = 291214597;
+  }
+
+  message ConstantSidePacket {
+    oneof value {
+      int32 int_value = 1;
+      float float_value = 2;
+      bool bool_value = 3;
+      string string_value = 4;
+    }
+  }
+
+  repeated ConstantSidePacket packet = 1;
+}

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -0,0 +1,196 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+template <typename T>
+void DoTestSingleSidePacket(absl::string_view packet_spec,
+                            const T& expected_value) {
+  static constexpr absl::string_view graph_config_template = R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet $0
+            }
+          }
+        }
+      )";
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          absl::Substitute(graph_config_template, packet_spec));
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("packet"));
+  auto actual_value =
+      graph.GetOutputSidePacket("packet").ValueOrDie().template Get<T>();
+  EXPECT_EQ(actual_value, expected_value);
+}
+
+TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
+  DoTestSingleSidePacket("{ int_value: 2 }", 2);
+  DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
+  DoTestSingleSidePacket("{ bool_value: true }", true);
+  DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+}
+
+TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "PACKET:1:float_packet"
+          output_side_packet: "PACKET:2:bool_packet"
+          output_side_packet: "PACKET:3:string_packet"
+          output_side_packet: "PACKET:4:another_string_packet"
+          output_side_packet: "PACKET:5:another_int_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+              packet { bool_value: false }
+              packet { string_value: "string" }
+              packet { string_value: "another string" }
+              packet { int_value: 128 }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
+            256);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+            0.5f);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
+  EXPECT_FALSE(
+      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "string");
+  MP_ASSERT_OK(graph.GetOutputSidePacket("another_string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("another_string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "another string");
+  MP_ASSERT_OK(graph.GetOutputSidePacket("another_int_packet"));
+  EXPECT_EQ(
+      graph.GetOutputSidePacket("another_int_packet").ValueOrDie().Get<int>(),
+      128);
+}
+
+TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "no_tag0"
+          output_side_packet: "PACKET:1:float_packet"
+          output_side_packet: "INCORRECT_TAG:0:name1"
+          output_side_packet: "PACKET:2:bool_packet"
+          output_side_packet: "PACKET:3:string_packet"
+          output_side_packet: "no_tag2"
+          output_side_packet: "INCORRECT_TAG:1:name2"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+              packet { bool_value: false }
+              packet { string_value: "string" }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
+            256);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+            0.5f);
+  MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
+  EXPECT_FALSE(
+      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
+  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
+                .ValueOrDie()
+                .Get<std::string>(),
+            "string");
+}
+
+TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:int_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+              packet { float_value: 0.5f }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  EXPECT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        node {
+          calculator: "ConstantSidePacketCalculator"
+          output_side_packet: "PACKET:0:int_packet"
+          output_side_packet: "PACKET:1:float_packet"
+          options: {
+            [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+              packet { int_value: 256 }
+            }
+          }
+        }
+      )");
+  CalculatorGraph graph;
+  EXPECT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -12,25 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cstdlib>
-#include <memory>
-#include <string>
+#include "mediapipe/calculators/core/packet_resampler_calculator.h"
 
-#include "absl/strings/str_cat.h"
-#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/deps/mathutil.h"
-#include "mediapipe/framework/deps/random_base.h"
-#include "mediapipe/framework/formats/video_stream_header.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/status_macros.h"
-#include "mediapipe/framework/tool/options_util.h"
+#include <memory>
 
 namespace {
+// Reflect an integer against the lower and upper bound of an interval.
+int64 ReflectBetween(int64 ts, int64 ts_min, int64 ts_max) {
+  if (ts < ts_min) return 2 * ts_min - ts - 1;
+  if (ts >= ts_max) return 2 * ts_max - ts - 1;
+  return ts;
+}
 
 // Creates a secure random number generator for use in ProcessWithJitter.
 // If no secure random number generator can be constructed, the jitter
@@ -45,120 +37,7 @@ std::unique_ptr<RandomBase> CreateSecureRandom(const std::string& seed) {
 
 namespace mediapipe {
 
-// This calculator is used to normalize the frequency of the packets
-// out of a stream. Given a desired frame rate, packets are going to be
-// removed or added to achieve it.
-//
-// The jitter feature is disabled by default. To enable it, you need to
-// implement CreateSecureRandom(const std::string&).
-//
-// The data stream may be either specified as the only stream (by index)
-// or as the stream with tag "DATA".
-//
-// The input and output streams may be accompanied by a VIDEO_HEADER
-// stream.  This stream includes a VideoHeader at Timestamp::PreStream().
-// The input VideoHeader on the VIDEO_HEADER stream will always be updated
-// with the resampler frame rate no matter what the options value for
-// output_header is before being output on the output VIDEO_HEADER stream.
-// If the input VideoHeader is not available, then only the frame rate
-// value will be set in the output.
-//
-// Related:
-//   packet_downsampler_calculator.cc: skips packets regardless of timestamps.
-class PacketResamplerCalculator : public CalculatorBase {
- public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  ::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
-
-  // Given the current count of periods that have passed, this returns
-  // the next valid timestamp of the middle point of the next period:
-  //    if count is 0, it returns the first_timestamp_.
-  //    if count is 1, it returns the first_timestamp_ + period (corresponding
-  //       to the first tick using exact fps)
-  // e.g. for frame_rate=30 and first_timestamp_=0:
-  //    0: 0
-  //    1: 33333
-  //    2: 66667
-  //    3: 100000
-  //
-  // Can only be used if jitter_ equals zero.
-  Timestamp PeriodIndexToTimestamp(int64 index) const;
-
-  // Given a Timestamp, finds the closest sync Timestamp based on
-  // first_timestamp_ and the desired fps.
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 TimestampToPeriodIndex(Timestamp timestamp) const;
-
-  // Outputs a packet if it is in range (start_time_, end_time_).
-  void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
-
-  // The timestamp of the first packet received.
-  Timestamp first_timestamp_;
-
-  // Number of frames per second (desired output frequency).
-  double frame_rate_;
-
-  // Inverse of frame_rate_.
-  int64 frame_time_usec_;
-
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
-  VideoHeader video_header_;
-  // The "DATA" input stream.
-  CollectionItemId input_data_id_;
-  // The "DATA" output stream.
-  CollectionItemId output_data_id_;
-
-  // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
-  bool flush_last_packet_;
-
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
-  double jitter_ = 0.0;
-  Timestamp next_output_timestamp_;
-
-  // If specified, output timestamps are aligned with base_timestamp.
-  // Otherwise, they are aligned with the first input timestamp.
-  Timestamp base_timestamp_;
-
-  // If specified, only outputs at/after start_time are included.
-  Timestamp start_time_;
-
-  // If specified, only outputs before end_time are included.
-  Timestamp end_time_;
-
-  // If set, the output timestamps nearest to start_time and end_time
-  // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.
-  bool round_limits_;
-};
-
 REGISTER_CALCULATOR(PacketResamplerCalculator);
-
 namespace {
 // Returns a TimestampDiff (assuming microseconds) corresponding to the
 // given time in seconds.
@@ -209,6 +88,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
+  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -239,6 +119,8 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
       << Timestamp::kTimestampUnitsPerSecond;
 
   frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
+  jitter_usec_ = static_cast<int64>(1000000.0 * jitter_ / frame_rate_);
+  RET_CHECK_LE(jitter_usec_, frame_time_usec_);
 
   video_header_.frame_rate = frame_rate_;
 
@@ -279,7 +161,10 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
           "SecureRandom is not available.  With \"jitter\" specified, "
           "PacketResamplerCalculator processing cannot proceed.");
     }
+    packet_reservoir_random_ = CreateSecureRandom(seed);
   }
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
   return ::mediapipe::OkStatus();
 }
 
@@ -294,6 +179,14 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
     }
   }
   if (jitter_ != 0.0 && random_ != nullptr) {
+    // Packet reservior is used to make sure there's an output for every period,
+    // e.g. partial period at the end of the stream.
+    if (packet_reservoir_->IsEnabled() &&
+        (first_timestamp_ == Timestamp::Unset() ||
+         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
+      packet_reservoir_->AddSample(curr_packet);
+    }
     MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
   } else {
     MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
@@ -303,11 +196,34 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 }
 
 void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
+  if (jitter_with_reflection_) {
+    next_output_timestamp_ =
+        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
+    return;
+  }
   next_output_timestamp_ =
       first_timestamp_ + frame_time_usec_ * random_->RandFloat();
 }
 
 void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  if (jitter_with_reflection_) {
+    next_output_timestamp_min_ += frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + frame_time_usec_;
+
+    next_output_timestamp_ += frame_time_usec_ +
+                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
+                              jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+    return;
+  }
+  packet_reservoir_->Disable();
   next_output_timestamp_ +=
       frame_time_usec_ *
       ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
@@ -339,10 +255,10 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
   while (true) {
     const int64 last_diff =
         (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-    RET_CHECK_GT(last_diff, 0.0);
+    RET_CHECK_GT(last_diff, 0);
     const int64 curr_diff =
         (next_output_timestamp_ - cc->InputTimestamp()).Value();
-    if (curr_diff > 0.0) {
+    if (curr_diff > 0) {
       break;
     }
     OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
@@ -431,6 +347,9 @@ void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
     OutputWithinLimits(cc,
                        last_packet_.At(PeriodIndexToTimestamp(period_count_)));
   }
+  if (!packet_reservoir_->IsEmpty()) {
+    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -0,0 +1,205 @@
+#ifndef MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/deps/mathutil.h"
+#include "mediapipe/framework/deps/random_base.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/tool/options_util.h"
+
+namespace mediapipe {
+
+class PacketReservoir {
+ public:
+  PacketReservoir(RandomBase* rng) : rng_(rng) {}
+  // Replace candidate with current packet with 1/count_ probability.
+  void AddSample(Packet sample) {
+    if (rng_->UnbiasedUniform(++count_) == 0) {
+      reservoir_ = sample;
+    }
+  }
+  bool IsEnabled() { return rng_ && enabled_; }
+  void Disable() {
+    if (enabled_) enabled_ = false;
+  }
+  void Clear() { count_ = 0; }
+  bool IsEmpty() { return count_ == 0; }
+  Packet GetSample() { return reservoir_; }
+
+ private:
+  RandomBase* rng_;
+  bool enabled_ = true;
+  int32 count_ = 0;
+  Packet reservoir_;
+};
+
+// This calculator is used to normalize the frequency of the packets
+// out of a stream. Given a desired frame rate, packets are going to be
+// removed or added to achieve it.
+//
+// If jitter_ is specified:
+//   - The first packet is chosen randomly (uniform distribution) among frames
+//     that correspond to timestamps [0, 1/frame_rate).  Let the chosen packet
+//     correspond to timestamp t.
+//   - The next packet is chosen randomly (uniform distribution) among frames
+//     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
+//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
+//       so that its marginal distribution is uniform within this interval.
+//       In the formula, t_0 is the timestamp of the first sampled
+//       packet, and the k is the packet index.
+//       See paper (https://arxiv.org/abs/2002.01147) for details.
+//   - t is updated and the process is repeated.
+//   - Note that seed is specified as input side packet for reproducibility of
+//     the resampling.  For Cloud ML Video Intelligence API, the hash of the
+//     input video should serve this purpose.  For YouTube, either video ID or
+//     content hex ID of the input video should do.
+//
+// If jitter_ is not specified:
+//   - The first packet defines the first_timestamp of the output stream,
+//     so it is always emitted.
+//   - If more packets are emitted, they will have timestamp equal to
+//     round(first_timestamp + k * period) , where k is a positive
+//     integer and the period is defined by the frame rate.
+//     Example: first_timestamp=0, fps=30, then the output stream
+//              will have timestamps: 0, 33333, 66667, 100000, etc...
+//   - The packets selected for the output stream are the ones closer
+//     to the exact middle point (33333.33, 66666.67 in our previous
+//     example). In case of ties, later packets are chosen.
+//   - 'Empty' periods happen when there are no packets for a long time
+//     (greater than a period). In this case, we send a copy of the last
+//     packet received before the empty period.
+// The jitter feature is disabled by default. To enable it, you need to
+// implement CreateSecureRandom(const std::string&).
+//
+// The data stream may be either specified as the only stream (by index)
+// or as the stream with tag "DATA".
+//
+// The input and output streams may be accompanied by a VIDEO_HEADER
+// stream.  This stream includes a VideoHeader at Timestamp::PreStream().
+// The input VideoHeader on the VIDEO_HEADER stream will always be updated
+// with the resampler frame rate no matter what the options value for
+// output_header is before being output on the output VIDEO_HEADER stream.
+// If the input VideoHeader is not available, then only the frame rate
+// value will be set in the output.
+//
+// Related:
+//   packet_downsampler_calculator.cc: skips packets regardless of timestamps.
+class PacketResamplerCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestampWithJitter();
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Logic for Process() when jitter_ != 0.0.
+  ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
+
+  // Logic for Process() when jitter_ == 0.0.
+  ::mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
+
+  // Given the current count of periods that have passed, this returns
+  // the next valid timestamp of the middle point of the next period:
+  //    if count is 0, it returns the first_timestamp_.
+  //    if count is 1, it returns the first_timestamp_ + period (corresponding
+  //       to the first tick using exact fps)
+  // e.g. for frame_rate=30 and first_timestamp_=0:
+  //    0: 0
+  //    1: 33333
+  //    2: 66667
+  //    3: 100000
+  //
+  // Can only be used if jitter_ equals zero.
+  Timestamp PeriodIndexToTimestamp(int64 index) const;
+
+  // Given a Timestamp, finds the closest sync Timestamp based on
+  // first_timestamp_ and the desired fps.
+  //
+  // Can only be used if jitter_ equals zero.
+  int64 TimestampToPeriodIndex(Timestamp timestamp) const;
+
+  // Outputs a packet if it is in range (start_time_, end_time_).
+  void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Number of frames per second (desired output frequency).
+  double frame_rate_;
+
+  // Inverse of frame_rate_.
+  int64 frame_time_usec_;
+
+  // Number of periods that have passed (= #packets sent to the output).
+  //
+  // Can only be used if jitter_ equals zero.
+  int64 period_count_;
+
+  // The last packet that was received.
+  Packet last_packet_;
+
+  VideoHeader video_header_;
+  // The "DATA" input stream.
+  CollectionItemId input_data_id_;
+  // The "DATA" output stream.
+  CollectionItemId output_data_id_;
+
+  // Indicator whether to flush last packet even if its timestamp is greater
+  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  bool flush_last_packet_;
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+  double jitter_ = 0.0;
+  bool jitter_with_reflection_;
+  int64 jitter_usec_;
+  Timestamp next_output_timestamp_;
+  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
+  // kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+
+  // If specified, only outputs at/after start_time are included.
+  Timestamp start_time_;
+
+  // If specified, only outputs before end_time are included.
+  Timestamp end_time_;
+
+  // If set, the output timestamps nearest to start_time and end_time
+  // are included in the output, even if the nearest timestamp is not
+  // between start_time and end_time.W
+  bool round_limits_;
+
+  // packet reservior used for sampling random packet out of partial
+  // period when jitter is enabled
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -66,6 +66,7 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+  optional bool jitter_with_reflection = 9 [default = false];
 
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/core/packet_resampler_calculator.h"
+
 #include <memory>
 #include <string>
 #include <vector>
@@ -29,7 +31,6 @@
 namespace mediapipe {
 
 namespace {
-
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
 // against expected outputs (both timestamps and contents).

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -0,0 +1,304 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Declaration of PacketThinnerCalculator.
+
+#include <cmath>  // for ceil
+#include <memory>
+
+#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+const double kTimebaseUs = 1000000;  // Microseconds.
+const char* const kPeriodTag = "PERIOD";
+}  // namespace
+
+// This calculator is used to thin an input stream of Packets.
+// An example application would be to sample decoded frames of video
+// at a coarser temporal resolution. Unless otherwise stated, all
+// timestamps are in units of microseconds.
+//
+// Thinning can be accomplished in one of two ways:
+// 1) asynchronous thinning (known below as async):
+//    Algorithm does not rely on a master clock and is parameterized only
+//    by a single option -- the period.  Once a packet is emitted, the
+//    thinner will discard subsequent packets for the duration of the period
+//    [Analogous to a refractory period during which packet emission is
+//    suppressed.]
+//    Packets arriving before start_time are discarded, as are packets
+//    arriving at or after end_time.
+// 2) synchronous thinning (known below as sync):
+//    There are two variants of this algorithm, both parameterized by a
+//    start_time and a period.  As in (1), packets arriving before start_time
+//    or at/after end_time are discarded.  Otherwise, at most one packet is
+//    emitted during a period, centered at timestamps generated by the
+//    expression:
+//      start_time + i * period  [where i is a non-negative integer]
+//    During each period, the packet closest to the generated timestamp is
+//    emitted (latest in the case of ties).  In the first variant
+//    (sync_output_timestamps = true), the emitted packet is output at the
+//    generated timestamp.  In the second variant, the packet is output at
+//    its original timestamp.  Both variants emit exactly the same packets,
+//    but at different timestamps.
+//
+// Thinning period can be provided in the calculator options or via a
+// side packet with the tag "PERIOD".
+//
+// Example config:
+// node {
+//   calculator: "PacketThinnerCalculator"
+//   input_stream: "signal"
+//   output_stream: "output"
+//   options {
+//     [mediapipe.PacketThinnerCalculatorOptions.ext] {
+//       thinner_type: SYNC
+//       period: 10
+//       sync_output_timestamps: true
+//       update_frame_rate: false
+//     }
+//   }
+// }
+class PacketThinnerCalculator : public CalculatorBase {
+ public:
+  PacketThinnerCalculator() {}
+  ~PacketThinnerCalculator() override {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
+    if (cc->InputSidePackets().HasTag(kPeriodTag)) {
+      cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (cc->InputTimestamp() < start_time_) {
+      return ::mediapipe::OkStatus();  // Drop packets before start_time_.
+    } else if (cc->InputTimestamp() >= end_time_) {
+      if (!cc->Outputs().Index(0).IsClosed()) {
+        cc->Outputs()
+            .Index(0)
+            .Close();  // No more Packets will be output after end_time_.
+      }
+      return ::mediapipe::OkStatus();
+    } else {
+      return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC
+                 ? AsyncThinnerProcess(cc)
+                 : SyncThinnerProcess(cc);
+    }
+  }
+
+ private:
+  // Implementation of ASYNC and SYNC versions of thinner algorithm.
+  ::mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
+  ::mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
+
+  // Cached option.
+  PacketThinnerCalculatorOptions::ThinnerType thinner_type_;
+
+  // Given a Timestamp, finds the closest sync Timestamp
+  // based on start_time_ and period_.  This can be earlier or
+  // later than given Timestamp, but is guaranteed to be within
+  // half a period_.
+  Timestamp NearestSyncTimestamp(Timestamp now) const;
+
+  // Cached option used by both async and sync thinners.
+  TimestampDiff period_;  // Interval during which only one packet is emitted.
+  Timestamp start_time_;  // Cached option - default Timestamp::Min()
+  Timestamp end_time_;    // Cached option - default Timestamp::Max()
+
+  // Only used by async thinner:
+  Timestamp next_valid_timestamp_;  // Suppress packets until this timestamp.
+
+  // Only used by sync thinner:
+  Packet saved_packet_;          // Best packet not yet emitted.
+  bool sync_output_timestamps_;  // Cached option.
+};
+REGISTER_CALCULATOR(PacketThinnerCalculator);
+
+namespace {
+TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
+}  // namespace
+
+::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
+  auto& options = cc->Options<PacketThinnerCalculatorOptions>();
+
+  thinner_type_ = options.thinner_type();
+  // This check enables us to assume only two thinner types exist in Process()
+  CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
+        thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
+      << "Unsupported thinner type.";
+
+  if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+    // ASYNC thinner outputs packets with the same timestamp as their input so
+    // its safe to SetOffset(0). SYNC thinner manipulates timestamps of its
+    // output so we don't do this for that case.
+    cc->SetOffset(0);
+  }
+
+  if (cc->InputSidePackets().HasTag(kPeriodTag)) {
+    period_ =
+        TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get<int64>());
+  } else {
+    period_ = TimestampDiff(options.period());
+  }
+  CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
+
+  if (options.has_start_time()) {
+    start_time_ = Timestamp(options.start_time());
+  } else if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+    start_time_ = Timestamp::Min();
+  } else {
+    start_time_ = Timestamp(0);
+  }
+
+  end_time_ =
+      options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
+  CHECK_LT(start_time_, end_time_)
+      << "Invalid PacketThinner: start_time must be earlier than end_time";
+
+  sync_output_timestamps_ = options.sync_output_timestamps();
+
+  next_valid_timestamp_ = start_time_;
+  // Drop packets until this time.
+  cc->Outputs().Index(0).SetNextTimestampBound(start_time_);
+
+  if (!cc->Inputs().Index(0).Header().IsEmpty()) {
+    if (options.update_frame_rate()) {
+      const VideoHeader& video_header =
+          cc->Inputs().Index(0).Header().Get<VideoHeader>();
+      double new_frame_rate;
+      if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
+        new_frame_rate =
+            video_header.frame_rate /
+            ceil(video_header.frame_rate * options.period() / kTimebaseUs);
+      } else {
+        const double sampling_rate = kTimebaseUs / options.period();
+        new_frame_rate = video_header.frame_rate < sampling_rate
+                             ? video_header.frame_rate
+                             : sampling_rate;
+      }
+      std::unique_ptr<VideoHeader> header(new VideoHeader);
+      header->format = video_header.format;
+      header->width = video_header.width;
+      header->height = video_header.height;
+      header->frame_rate = new_frame_rate;
+      cc->Outputs().Index(0).SetHeader(Adopt(header.release()));
+    } else {
+      cc->Outputs().Index(0).SetHeader(cc->Inputs().Index(0).Header());
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
+  // Emit any saved packets before quitting.
+  if (!saved_packet_.IsEmpty()) {
+    // Only sync thinner should have saved packets.
+    CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
+    if (sync_output_timestamps_) {
+      cc->Outputs().Index(0).AddPacket(
+          saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
+    } else {
+      cc->Outputs().Index(0).AddPacket(saved_packet_);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
+    CalculatorContext* cc) {
+  if (cc->InputTimestamp() >= next_valid_timestamp_) {
+    cc->Outputs().Index(0).AddPacket(
+        cc->Inputs().Index(0).Value());  // Emit current packet.
+    next_valid_timestamp_ = cc->InputTimestamp() + period_;
+    // Guaranteed not to emit packets seen during refractory period.
+    cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
+    CalculatorContext* cc) {
+  if (saved_packet_.IsEmpty()) {
+    // If no packet has been saved, store the current packet.
+    saved_packet_ = cc->Inputs().Index(0).Value();
+    cc->Outputs().Index(0).SetNextTimestampBound(
+        sync_output_timestamps_ ? NearestSyncTimestamp(cc->InputTimestamp())
+                                : cc->InputTimestamp());
+  } else {
+    // Saved packet exists -- update or emit.
+    const Timestamp saved = saved_packet_.Timestamp();
+    const Timestamp saved_sync = NearestSyncTimestamp(saved);
+    const Timestamp now = cc->InputTimestamp();
+    const Timestamp now_sync = NearestSyncTimestamp(now);
+    CHECK_LE(saved_sync, now_sync);
+    if (saved_sync == now_sync) {
+      // Saved Packet is in same interval as current packet.
+      // Replace saved packet with current if it is at least as
+      // central as the saved packet wrt temporal interval.
+      // [We break ties in favor of fresher packets]
+      if (abs(now - now_sync) <= abs(saved - saved_sync)) {
+        saved_packet_ = cc->Inputs().Index(0).Value();
+      }
+    } else {
+      // Saved packet is the best packet from earlier interval: emit!
+      if (sync_output_timestamps_) {
+        cc->Outputs().Index(0).AddPacket(saved_packet_.At(saved_sync));
+        cc->Outputs().Index(0).SetNextTimestampBound(now_sync);
+      } else {
+        cc->Outputs().Index(0).AddPacket(saved_packet_);
+        cc->Outputs().Index(0).SetNextTimestampBound(now);
+      }
+      // Current packet is the first one we've seen from new interval -- save!
+      saved_packet_ = cc->Inputs().Index(0).Value();
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
+Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
+  CHECK_NE(start_time_, Timestamp::Unset())
+      << "Method only valid for sync thinner calculator.";
+
+  // Computation is done using int64 arithmetic.  No easy way to avoid
+  // since Timestamps don't support div and multiply.
+  const int64 now64 = now.Value();
+  const int64 start64 = start_time_.Value();
+  const int64 period64 = period_.Value();
+  CHECK_LE(0, period64);
+
+  // Round now64 to its closest interval (units of period64).
+  int64 sync64 =
+      (now64 - start64 + period64 / 2) / period64 * period64 + start64;
+  CHECK_LE(abs(now64 - sync64), period64 / 2)
+      << "start64: " << start64 << "; now64: " << now64
+      << "; sync64: " << sync64;
+
+  return Timestamp(sync64);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_thinner_calculator.proto

@@ -0,0 +1,66 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message PacketThinnerCalculatorOptions {
+  extend CalculatorOptions {
+    optional PacketThinnerCalculatorOptions ext = 288533508;
+  }
+
+  enum ThinnerType {
+    ASYNC = 1;  // Asynchronous thinner, described below [default].
+    SYNC = 2;   // Synchronous thinner, also described below.
+  }
+  optional ThinnerType thinner_type = 1 [default = ASYNC];
+
+  // The period (in microsecond) specifies the temporal interval during which
+  // only a single packet is emitted in the output stream.  Has subtly different
+  // semantics depending on the thinner type, as follows.
+  //
+  // Async thinner: this option is a refractory period -- once a packet is
+  // emitted, we guarantee that no packets will be emitted for period ticks.
+  //
+  // Sync thinner: the period specifies a temporal interval during which
+  // only one packet is emitted.  The emitted packet is guaranteed to be
+  // the one closest to the center of the temporal interval (no guarantee on
+  // how ties are broken).  More specifically,
+  //   intervals are centered at start_time + i * period
+  //   (for non-negative integers i).
+  // Thus, each interval extends period/2 ticks before and after its center.
+  // Additionally, in the sync thinner any packets earlier than start_time
+  // are discarded and the thinner calls Close() once timestamp equals or
+  // exceeds end_time.
+  optional int64 period = 2 [default = 1];
+
+  // Packets before start_time and at/after end_time are discarded.
+  // Additionally, for a sync thinner, start time specifies the center of
+  // time invervals as described above and therefore should be set explicitly.
+  optional int64 start_time = 3;  // If not specified, set to 0 for SYNC type,
+                                  // and set to Timestamp::Min() for ASYNC type.
+  optional int64 end_time = 4;    // Set to Timestamp::Max() if not specified.
+
+  // Whether the timestamps of packets emitted by sync thinner should
+  // correspond to the center of their corresponding temporal interval.
+  // If false, packets emitted using original timestamp (as in async thinner).
+  optional bool sync_output_timestamps = 5 [default = true];
+
+  // If true, update the frame rate in the header, if it's available, to an
+  // estimated frame rate due to the sampling.
+  optional bool update_frame_rate = 6 [default = false];
+}

mediapipe/calculators/core/packet_thinner_calculator_test.cc

@@ -0,0 +1,357 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+// A simple version of CalculatorRunner with built-in convenience methods for
+// setting inputs from a vector and checking outputs against a vector of
+// expected outputs.
+class SimpleRunner : public CalculatorRunner {
+ public:
+  explicit SimpleRunner(const CalculatorOptions& options)
+      : CalculatorRunner("PacketThinnerCalculator", options) {
+    SetNumInputs(1);
+    SetNumOutputs(1);
+    SetNumInputSidePackets(0);
+  }
+
+  explicit SimpleRunner(const CalculatorGraphConfig::Node& node)
+      : CalculatorRunner(node) {}
+
+  void SetInput(const std::vector<int>& timestamp_list) {
+    MutableInputs()->Index(0).packets.clear();
+    for (const int ts : timestamp_list) {
+      MutableInputs()->Index(0).packets.push_back(
+          MakePacket<std::string>(absl::StrCat("Frame #", ts))
+              .At(Timestamp(ts)));
+    }
+  }
+
+  void SetFrameRate(const double frame_rate) {
+    auto video_header = absl::make_unique<VideoHeader>();
+    video_header->frame_rate = frame_rate;
+    MutableInputs()->Index(0).header = Adopt(video_header.release());
+  }
+
+  std::vector<int64> GetOutputTimestamps() const {
+    std::vector<int64> timestamps;
+    for (const Packet& packet : Outputs().Index(0).packets) {
+      timestamps.emplace_back(packet.Timestamp().Value());
+    }
+    return timestamps;
+  }
+
+  double GetFrameRate() const {
+    CHECK(!Outputs().Index(0).header.IsEmpty());
+    return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
+  }
+};
+
+// Check that thinner respects start_time and end_time options.
+// We only test with one thinner because the logic for start & end time
+// handling is shared across both types of thinner in Process().
+TEST(PacketThinnerCalculatorTest, StartAndEndTimeTest) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_start_time(4);
+  extension->set_end_time(12);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {5, 11};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, AsyncUniformStreamThinningTest) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("PacketThinnerCalculator");
+  node.add_input_side_packet("PERIOD:period");
+  node.add_input_stream("input_stream");
+  node.add_output_stream("output_stream");
+  auto* extension = node.mutable_options()->MutableExtension(
+      PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_start_time(0);
+  extension->set_sync_output_timestamps(false);
+
+  SimpleRunner runner(node);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest1) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("PacketThinnerCalculator");
+  node.add_input_side_packet("PERIOD:period");
+  node.add_input_stream("input_stream");
+  node.add_output_stream("output_stream");
+  auto* extension = node.mutable_options()->MutableExtension(
+      PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_sync_output_timestamps(false);
+
+  SimpleRunner runner(node);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest2) {
+  // Same test but now with synced timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+// Test: Given a stream with timestamps corresponding to first ten prime numbers
+// and period of 5, confirm whether timestamps of thinner stream matches
+// expectations.
+TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest1) {
+  // ASYNC thinner.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 7, 13, 19, 29};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest2) {
+  // SYNC with original timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 5, 11, 17, 19, 23, 29};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+// Confirm that Calculator correctly handles boundary cases.
+TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest1) {
+  // Odd period, negative start_time
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(-10);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  // Two timestamps falling on either side of a period boundary.
+  runner.SetInput({2, 3});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest2) {
+  // Even period, negative start_time, negative packet timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(-144);
+  extension->set_period(6);
+  extension->set_sync_output_timestamps(true);
+  SimpleRunner runner(options);
+  // Two timestamps falling on either side of a period boundary.
+  runner.SetInput({-4, -3, 8, 9});
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {-6, 0, 6, 12};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest1) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 8, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true sampling period is 6.
+  EXPECT_DOUBLE_EQ(1000000.0 / 6, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest2) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC);
+  extension->set_period(5);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({8, 16, 24, 32, 40, 48, 56});
+  runner.SetFrameRate(1000000.0 / 8);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<int64> expected_timestamps = {8, 16, 24, 32, 40, 48, 56};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true sampling period is still 8.
+  EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest3) {
+  // Note: sync runner but outputting *original* timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(false);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {2, 6, 10, 14};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 5.
+  EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest4) {
+  // Same test but now with synced timestamps.
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({2, 4, 6, 8, 10, 12, 14});
+  runner.SetFrameRate(1000000.0 / 2);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {0, 5, 10, 15};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 5.
+  EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate());
+}
+
+TEST(PacketThinnerCalculatorTest, FrameRateTest5) {
+  CalculatorOptions options;
+  auto* extension =
+      options.MutableExtension(PacketThinnerCalculatorOptions::ext);
+  extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC);
+  extension->set_start_time(0);
+  extension->set_period(5);
+  extension->set_sync_output_timestamps(true);
+  extension->set_update_frame_rate(true);
+  SimpleRunner runner(options);
+  runner.SetInput({8, 16, 24, 32, 40, 48, 56});
+  runner.SetFrameRate(1000000.0 / 8);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<int64> expected_timestamps = {10, 15, 25, 30, 40, 50, 55};
+  EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps());
+  // The true (long-run) sampling period is 8.
+  EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -17,6 +17,7 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
 
@@ -86,6 +87,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
         main_ts_.pop_front();
       }
     }
+    auto& loop_out = cc->Outputs().Get(loop_out_id_);
 
     while (!main_ts_.empty() && !loopback_packets_.empty()) {
       Timestamp main_timestamp = main_ts_.front();
@@ -95,18 +97,31 @@ class PreviousLoopbackCalculator : public CalculatorBase {
 
       if (previous_loopback.IsEmpty()) {
         // TODO: SetCompleteTimestampBound would be more useful.
-        cc->Outputs()
-            .Get(loop_out_id_)
-            .SetNextTimestampBound(main_timestamp + 1);
+        loop_out.SetNextTimestampBound(main_timestamp + 1);
       } else {
-        cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
+        loop_out.AddPacket(std::move(previous_loopback));
+      }
+    }
+
+    // In case of an empty loopback input, the next timestamp bound for
+    // loopback input is the loopback timestamp + 1.  The next timestamp bound
+    // for output is set and the main_ts_ vector is truncated accordingly.
+    if (loopback_packet.IsEmpty() &&
+        loopback_packet.Timestamp() != Timestamp::Unstarted()) {
+      Timestamp loopback_bound =
+          loopback_packet.Timestamp().NextAllowedInStream();
+      while (!main_ts_.empty() && main_ts_.front() <= loopback_bound) {
+        main_ts_.pop_front();
+      }
+      if (main_ts_.empty()) {
+        loop_out.SetNextTimestampBound(loopback_bound.NextAllowedInStream());
       }
     }
     if (!main_ts_.empty()) {
-      cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
+      loop_out.SetNextTimestampBound(main_ts_.front());
     }
     if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
-      cc->Outputs().Get(loop_out_id_).Close();
+      loop_out.Close();
     }
     return ::mediapipe::OkStatus();
   }

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -207,5 +207,64 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
+// Demonstrates that downstream calculators won't be blocked by
+// always-empty-LOOP-stream.
+TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
+  std::vector<Packet> outputs;
+  CalculatorGraphConfig graph_config_ =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: 'in'
+        node {
+          calculator: 'PreviousLoopbackCalculator'
+          input_stream: 'MAIN:in'
+          input_stream: 'LOOP:previous'
+          input_stream_info: { tag_index: 'LOOP' back_edge: true }
+          output_stream: 'PREV_LOOP:previous'
+        }
+        # This calculator synchronizes its inputs as normal, so it is used
+        # to check that both "in" and "previous" are ready.
+        node {
+          calculator: 'PassThroughCalculator'
+          input_stream: 'in'
+          input_stream: 'previous'
+          output_stream: 'out'
+          output_stream: 'previous2'
+        }
+        node {
+          calculator: 'PacketOnCloseCalculator'
+          input_stream: 'out'
+          output_stream: 'close_out'
+        }
+      )");
+  tool::AddVectorSink("close_out", &graph_config_, &outputs);
+
+  CalculatorGraph graph_;
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.StartRun({}));
+
+  auto send_packet = [&graph_](const std::string& input_name, int n) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        input_name, MakePacket<int>(n).At(Timestamp(n))));
+  };
+
+  send_packet("in", 0);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
+
+  for (int main_ts = 1; main_ts < 50; ++main_ts) {
+    send_packet("in", main_ts);
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+    std::vector<int64> ts_values = TimestampValues(outputs);
+    EXPECT_EQ(ts_values.size(), main_ts + 1);
+    for (int j = 0; j < main_ts; ++j) {
+      EXPECT_EQ(ts_values[j], j);
+    }
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
 }  // anonymous namespace
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -20,6 +20,10 @@
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
 namespace mediapipe {
 
 // Example config:
@@ -36,14 +40,21 @@ namespace mediapipe {
 //     }
 //   }
 // }
-typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
+typedef SplitVectorCalculator<TfLiteTensor, false>
+    SplitTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
 
-typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
+typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark, false>
     SplitLandmarkVectorCalculator;
 REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
 
-typedef SplitVectorCalculator<::mediapipe::NormalizedRect>
+typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
     SplitNormalizedRectVectorCalculator;
 REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
+    MovableSplitGlBufferVectorCalculator;
+REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator);
+#endif
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.h

@@ -15,12 +15,14 @@
 #ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
 
+#include <type_traits>
 #include <vector>
 
 #include "mediapipe/calculators/core/split_vector_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/interpreter.h"
@@ -29,6 +31,20 @@
 
 namespace mediapipe {
 
+template <typename T>
+using IsCopyable = std::enable_if_t<std::is_copy_constructible<T>::value, bool>;
+
+template <typename T>
+using IsNotCopyable =
+    std::enable_if_t<!std::is_copy_constructible<T>::value, bool>;
+
+template <typename T>
+using IsMovable = std::enable_if_t<std::is_move_constructible<T>::value, bool>;
+
+template <typename T>
+using IsNotMovable =
+    std::enable_if_t<!std::is_move_constructible<T>::value, bool>;
+
 // Splits an input packet with std::vector<T> into multiple std::vector<T>
 // output packets using the [begin, end) ranges specified in
 // SplitVectorCalculatorOptions. If the option "element_only" is set to true,
@@ -39,7 +55,7 @@ namespace mediapipe {
 // combined into one vector.
 // To use this class for a particular type T, register a calculator using
 // SplitVectorCalculator<T>.
-template <typename T>
+template <typename T, bool move_elements>
 class SplitVectorCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -51,23 +67,16 @@ class SplitVectorCalculator : public CalculatorBase {
     const auto& options =
         cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
 
+    if (!std::is_copy_constructible<T>::value || move_elements) {
+      // Ranges of elements shouldn't overlap when the vector contains
+      // non-copyable elements.
+      RET_CHECK_OK(checkRangesDontOverlap(options));
+    }
+
     if (options.combine_outputs()) {
       RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
       cc->Outputs().Index(0).Set<std::vector<T>>();
-      for (int i = 0; i < options.ranges_size() - 1; ++i) {
-        for (int j = i + 1; j < options.ranges_size(); ++j) {
-          const auto& range_0 = options.ranges(i);
-          const auto& range_1 = options.ranges(j);
-          if ((range_0.begin() >= range_1.begin() &&
-               range_0.begin() < range_1.end()) ||
-              (range_1.begin() >= range_0.begin() &&
-               range_1.begin() < range_0.end())) {
-            return ::mediapipe::InvalidArgumentError(
-                "Ranges must be non-overlapping when using combine_outputs "
-                "option.");
-          }
-        }
-      }
+      RET_CHECK_OK(checkRangesDontOverlap(options));
     } else {
       if (cc->Outputs().NumEntries() != options.ranges_size()) {
         return ::mediapipe::InvalidArgumentError(
@@ -117,14 +126,26 @@ class SplitVectorCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
-    const auto& input = cc->Inputs().Index(0).Get<std::vector<T>>();
-    RET_CHECK_GE(input.size(), max_range_end_);
+    if (cc->Inputs().Index(0).IsEmpty()) return ::mediapipe::OkStatus();
 
+    if (move_elements) {
+      return ProcessMovableElements<T>(cc);
+    } else {
+      return ProcessCopyableElements<T>(cc);
+    }
+  }
+
+  template <typename U, IsCopyable<U> = true>
+  ::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
+    // static_assert(std::is_copy_constructible<U>::value,
+    //              "Cannot copy non-copyable elements");
+    const auto& input = cc->Inputs().Index(0).Get<std::vector<U>>();
+    RET_CHECK_GE(input.size(), max_range_end_);
     if (combine_outputs_) {
-      auto output = absl::make_unique<std::vector<T>>();
+      auto output = absl::make_unique<std::vector<U>>();
       output->reserve(total_elements_);
       for (int i = 0; i < ranges_.size(); ++i) {
-        auto elements = absl::make_unique<std::vector<T>>(
+        auto elements = absl::make_unique<std::vector<U>>(
             input.begin() + ranges_[i].first,
             input.begin() + ranges_[i].second);
         output->insert(output->end(), elements->begin(), elements->end());
@@ -134,7 +155,7 @@ class SplitVectorCalculator : public CalculatorBase {
       if (element_only_) {
         for (int i = 0; i < ranges_.size(); ++i) {
           cc->Outputs().Index(i).AddPacket(
-              MakePacket<T>(input[ranges_[i].first]).At(cc->InputTimestamp()));
+              MakePacket<U>(input[ranges_[i].first]).At(cc->InputTimestamp()));
         }
       } else {
         for (int i = 0; i < ranges_.size(); ++i) {
@@ -149,7 +170,78 @@ class SplitVectorCalculator : public CalculatorBase {
     return ::mediapipe::OkStatus();
   }
 
+  template <typename U, IsNotCopyable<U> = true>
+  ::mediapipe::Status ProcessCopyableElements(CalculatorContext* cc) {
+    return ::mediapipe::InternalError("Cannot copy non-copyable elements.");
+  }
+
+  template <typename U, IsMovable<U> = true>
+  ::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
+    ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+        cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
+    if (!input_status.ok()) return input_status.status();
+    std::unique_ptr<std::vector<U>> input_vector =
+        std::move(input_status).ValueOrDie();
+    RET_CHECK_GE(input_vector->size(), max_range_end_);
+
+    if (combine_outputs_) {
+      auto output = absl::make_unique<std::vector<U>>();
+      output->reserve(total_elements_);
+      for (int i = 0; i < ranges_.size(); ++i) {
+        output->insert(
+            output->end(),
+            std::make_move_iterator(input_vector->begin() + ranges_[i].first),
+            std::make_move_iterator(input_vector->begin() + ranges_[i].second));
+      }
+      cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<U>(std::move(input_vector->at(ranges_[i].first)))
+                  .At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          auto output = absl::make_unique<std::vector<T>>();
+          output->insert(
+              output->end(),
+              std::make_move_iterator(input_vector->begin() + ranges_[i].first),
+              std::make_move_iterator(input_vector->begin() +
+                                      ranges_[i].second));
+          cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U, IsNotMovable<U> = true>
+  ::mediapipe::Status ProcessMovableElements(CalculatorContext* cc) {
+    return ::mediapipe::InternalError("Cannot move non-movable elements.");
+  }
+
  private:
+  static ::mediapipe::Status checkRangesDontOverlap(
+      const ::mediapipe::SplitVectorCalculatorOptions& options) {
+    for (int i = 0; i < options.ranges_size() - 1; ++i) {
+      for (int j = i + 1; j < options.ranges_size(); ++j) {
+        const auto& range_0 = options.ranges(i);
+        const auto& range_1 = options.ranges(j);
+        if ((range_0.begin() >= range_1.begin() &&
+             range_0.begin() < range_1.end()) ||
+            (range_1.begin() >= range_0.begin() &&
+             range_1.begin() < range_0.end())) {
+          return ::mediapipe::InvalidArgumentError(
+              "Ranges must be non-overlapping when using combine_outputs "
+              "option.");
+        }
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
   std::vector<std::pair<int32, int32>> ranges_;
   int32 max_range_end_ = -1;
   int32 total_elements_ = 0;

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -452,4 +452,243 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   ASSERT_FALSE(graph.Initialize(graph_config).ok());
 }
 
+typedef SplitVectorCalculator<std::unique_ptr<int>, true>
+    MovableSplitUniqueIntPtrCalculator;
+REGISTER_CALCULATOR(MovableSplitUniqueIntPtrCalculator);
+
+class MovableSplitUniqueIntPtrCalculatorTest : public ::testing::Test {
+ protected:
+  void ValidateVectorOutput(std::vector<Packet>& output_packets,
+                            int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const std::vector<std::unique_ptr<int>>& output_vec =
+        output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
+    ASSERT_EQ(expected_elements, output_vec.size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const int expected_value = input_begin_index + i;
+      const std::unique_ptr<int>& result = output_vec[i];
+      ASSERT_NE(result, nullptr);
+      ASSERT_EQ(expected_value, *result);
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int expected_value) {
+    ASSERT_EQ(1, output_packets.size());
+    const std::unique_ptr<int>& result =
+        output_packets[0].Get<std::unique_ptr<int>>();
+    ASSERT_NE(result, nullptr);
+    ASSERT_EQ(expected_value, *result);
+  }
+
+  void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
+                                    int expected_elements,
+                                    std::vector<int>& input_begin_indices,
+                                    std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const std::vector<std::unique_ptr<int>>& output_vector =
+        output_packets[0].Get<std::vector<std::unique_ptr<int>>>();
+    ASSERT_EQ(expected_elements, output_vector.size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const int expected_value = i;
+        const std::unique_ptr<int>& result = output_vector[element_id];
+        ASSERT_NE(result, nullptr);
+        ASSERT_EQ(expected_value, *result);
+        ++element_id;
+      }
+    }
+  }
+};
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/0);
+  ValidateVectorOutput(range_1_packets, /*expected_elements=*/3,
+                       /*input_begin_index=*/1);
+  ValidateVectorOutput(range_2_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/4);
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ValidateElementOutput(range_0_packets, /*expected_value=*/0);
+  ValidateElementOutput(range_1_packets, /*expected_value=*/2);
+  ValidateElementOutput(range_2_packets, /*expected_value=*/4);
+}
+
+TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
+  // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "input_vector"
+            node {
+              calculator: "MovableSplitUniqueIntPtrCalculator"
+              input_stream: "input_vector"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // input_vector : {0, 1, 2, 3, 4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_vector =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
+                               input_begin_indices, input_end_indices);
+}
+
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -80,7 +80,9 @@ mediapipe_cc_proto_library(
     name = "opencv_image_encoder_calculator_cc_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//visibility:public",
+    ],
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
@@ -330,6 +332,7 @@ cc_library(
 cc_library(
     name = "image_cropping_calculator",
     srcs = ["image_cropping_calculator.cc"],
+    hdrs = ["image_cropping_calculator.h"],
     copts = select({
         "//mediapipe:apple": [
             "-x objective-c++",
@@ -369,6 +372,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "image_cropping_calculator_test",
+    srcs = ["image_cropping_calculator_test.cc"],
+    deps = [
+        ":image_cropping_calculator",
+        ":image_cropping_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:tag_map",
+        "//mediapipe/framework/tool:tag_map_helper",
+    ],
+)
+
 cc_library(
     name = "luminance_calculator",
     srcs = ["luminance_calculator.cc"],

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/image/image_cropping_calculator.h"
+
 #include <cmath>
 
-#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
-#include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/rect.pb.h"
@@ -25,7 +25,6 @@
 #include "mediapipe/framework/port/status.h"
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-#include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/shader_util.h"
@@ -52,62 +51,6 @@ constexpr char kWidthTag[] = "WIDTH";
 
 }  // namespace
 
-// Crops the input texture to the given rectangle region. The rectangle can
-// be at arbitrary location on the image with rotation. If there's rotation, the
-// output texture will have the size of the input rectangle. The rotation should
-// be in radian, see rect.proto for detail.
-//
-// Input:
-//   One of the following two tags:
-//   IMAGE - ImageFrame representing the input image.
-//   IMAGE_GPU - GpuBuffer representing the input image.
-//   One of the following two tags (optional if WIDTH/HEIGHT is specified):
-//   RECT - A Rect proto specifying the width/height and location of the
-//          cropping rectangle.
-//   NORM_RECT - A NormalizedRect proto specifying the width/height and location
-//               of the cropping rectangle in normalized coordinates.
-//   Alternative tags to RECT (optional if RECT/NORM_RECT is specified):
-//   WIDTH - The desired width of the output cropped image,
-//           based on image center
-//   HEIGHT - The desired height of the output cropped image,
-//            based on image center
-//
-// Output:
-//   One of the following two tags:
-//   IMAGE - Cropped ImageFrame
-//   IMAGE_GPU - Cropped GpuBuffer.
-//
-// Note: input_stream values take precedence over options defined in the graph.
-//
-class ImageCroppingCalculator : public CalculatorBase {
- public:
-  ImageCroppingCalculator() = default;
-  ~ImageCroppingCalculator() override = default;
-
-  static ::mediapipe::Status GetContract(CalculatorContract* cc);
-  ::mediapipe::Status Open(CalculatorContext* cc) override;
-  ::mediapipe::Status Process(CalculatorContext* cc) override;
-  ::mediapipe::Status Close(CalculatorContext* cc) override;
-
- private:
-  ::mediapipe::Status RenderCpu(CalculatorContext* cc);
-  ::mediapipe::Status RenderGpu(CalculatorContext* cc);
-  ::mediapipe::Status InitGpu(CalculatorContext* cc);
-  void GlRender();
-  void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
-                           int* dst_width, int* dst_height);
-
-  mediapipe::ImageCroppingCalculatorOptions options_;
-
-  bool use_gpu_ = false;
-  // Output texture corners (4) after transoformation in normalized coordinates.
-  float transformed_points_[8];
-#if !defined(MEDIAPIPE_DISABLE_GPU)
-  bool gpu_initialized_ = false;
-  mediapipe::GlCalculatorHelper gpu_helper_;
-  GLuint program_ = 0;
-#endif  //  !MEDIAPIPE_DISABLE_GPU
-};
 REGISTER_CALCULATOR(ImageCroppingCalculator);
 
 ::mediapipe::Status ImageCroppingCalculator::GetContract(
@@ -132,7 +75,11 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag));
+  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag) ^
+            (cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
+                 .has_norm_width() &&
+             cc->Options<mediapipe::ImageCroppingCalculatorOptions>()
+                 .has_norm_height()));
   if (cc->Inputs().HasTag(kRectTag)) {
     cc->Inputs().Tag(kRectTag).Set<Rect>();
   }
@@ -222,41 +169,8 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
-  float rect_center_x = input_img.Width() / 2.0f;
-  float rect_center_y = input_img.Height() / 2.0f;
-  float rotation = 0.0f;
-  int target_width = input_img.Width();
-  int target_height = input_img.Height();
-  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
-      rect_center_x = rect.x_center();
-      rect_center_y = rect.y_center();
-      target_width = rect.width();
-      target_height = rect.height();
-      rotation = rect.rotation();
-    }
-  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
-    if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
-        rect.y_center() >= 0.0) {
-      rect_center_x = std::round(rect.x_center() * input_img.Width());
-      rect_center_y = std::round(rect.y_center() * input_img.Height());
-      target_width = std::round(rect.width() * input_img.Width());
-      target_height = std::round(rect.height() * input_img.Height());
-      rotation = rect.rotation();
-    }
-  } else {
-    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      target_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      target_height = cc->Inputs().Tag(kHeightTag).Get<int>();
-    } else if (options_.has_width() && options_.has_height()) {
-      target_width = options_.width();
-      target_height = options_.height();
-    }
-    rotation = options_.rotation();
-  }
+  auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
+      GetCropSpecs(cc, input_img.Width(), input_img.Height());
 
   const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
                                  cv::Size2f(target_width, target_height),
@@ -433,46 +347,8 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
                                                   int src_width, int src_height,
                                                   int* dst_width,
                                                   int* dst_height) {
-  // Get the size of the cropping box.
-  int crop_width = src_width;
-  int crop_height = src_height;
-  // Get the center of cropping box. Default is the at the center.
-  int x_center = src_width / 2;
-  int y_center = src_height / 2;
-  // Get the rotation of the cropping box.
-  float rotation = 0.0f;
-  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
-    // Only use the rect if it is valid.
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
-      x_center = rect.x_center();
-      y_center = rect.y_center();
-      crop_width = rect.width();
-      crop_height = rect.height();
-      rotation = rect.rotation();
-    }
-  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
-    // Only use the rect if it is valid.
-    if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
-        rect.y_center() >= 0.0) {
-      x_center = std::round(rect.x_center() * src_width);
-      y_center = std::round(rect.y_center() * src_height);
-      crop_width = std::round(rect.width() * src_width);
-      crop_height = std::round(rect.height() * src_height);
-      rotation = rect.rotation();
-    }
-  } else {
-    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
-    } else if (options_.has_width() && options_.has_height()) {
-      crop_width = options_.width();
-      crop_height = options_.height();
-    }
-    rotation = options_.rotation();
-  }
+  auto [crop_width, crop_height, x_center, y_center, rotation] =
+      GetCropSpecs(cc, src_width, src_height);
 
   const float half_width = crop_width / 2.0f;
   const float half_height = crop_height / 2.0f;
@@ -508,4 +384,82 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
   *dst_height = std::max(1, height);
 }
 
+RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
+                                               int src_width, int src_height) {
+  // Get the size of the cropping box.
+  int crop_width = src_width;
+  int crop_height = src_height;
+  // Get the center of cropping box. Default is the at the center.
+  int x_center = src_width / 2;
+  int y_center = src_height / 2;
+  // Get the rotation of the cropping box.
+  float rotation = 0.0f;
+  // Get the normalized width and height if specified by the inputs or options.
+  float normalized_width = 0.0f;
+  float normalized_height = 0.0f;
+
+  mediapipe::ImageCroppingCalculatorOptions options =
+      cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
+
+  // width/height, norm_width/norm_height from input streams take precednece.
+  if (cc->Inputs().HasTag(kRectTag)) {
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
+    // Only use the rect if it is valid.
+    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
+        rect.y_center() >= 0) {
+      x_center = rect.x_center();
+      y_center = rect.y_center();
+      crop_width = rect.width();
+      crop_height = rect.height();
+      rotation = rect.rotation();
+    }
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
+    const auto& norm_rect =
+        cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
+    if (norm_rect.width() > 0.0 && norm_rect.height() > 0.0) {
+      normalized_width = norm_rect.width();
+      normalized_height = norm_rect.height();
+      x_center = std::round(norm_rect.x_center() * src_width);
+      y_center = std::round(norm_rect.y_center() * src_height);
+      rotation = norm_rect.rotation();
+    }
+  } else if (cc->Inputs().HasTag(kWidthTag) &&
+             cc->Inputs().HasTag(kHeightTag)) {
+    crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
+    crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
+  } else if (options.has_width() && options.has_height()) {
+    crop_width = options.width();
+    crop_height = options.height();
+  } else if (options.has_norm_width() && options.has_norm_height()) {
+    normalized_width = options.norm_width();
+    normalized_height = options.norm_height();
+  }
+
+  // Get the crop width and height from the normalized width and height.
+  if (normalized_width > 0 && normalized_height > 0) {
+    crop_width = std::round(normalized_width * src_width);
+    crop_height = std::round(normalized_height * src_height);
+  }
+
+  // Rotation and center values from input streams take precedence, so only
+  // look at those values in the options if kRectTag and kNormRectTag are not
+  // present from the inputs.
+  if (!cc->Inputs().HasTag(kRectTag) && !cc->Inputs().HasTag(kNormRectTag)) {
+    if (options.has_norm_center_x() && options.has_norm_center_y()) {
+      x_center = std::round(options.norm_center_x() * src_width);
+      y_center = std::round(options.norm_center_y() * src_height);
+    }
+    if (options.has_rotation()) {
+      rotation = options.rotation();
+    }
+  }
+  return {
+      .width = crop_width,
+      .height = crop_height,
+      .center_x = x_center,
+      .center_y = y_center,
+      .rotation = rotation,
+  };
+}
+
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.h

@@ -0,0 +1,86 @@
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_
+
+#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+// Crops the input texture to the given rectangle region. The rectangle can
+// be at arbitrary location on the image with rotation. If there's rotation, the
+// output texture will have the size of the input rectangle. The rotation should
+// be in radian, see rect.proto for detail.
+//
+// Input:
+//   One of the following two tags:
+//   IMAGE - ImageFrame representing the input image.
+//   IMAGE_GPU - GpuBuffer representing the input image.
+//   One of the following two tags (optional if WIDTH/HEIGHT is specified):
+//   RECT - A Rect proto specifying the width/height and location of the
+//          cropping rectangle.
+//   NORM_RECT - A NormalizedRect proto specifying the width/height and location
+//               of the cropping rectangle in normalized coordinates.
+//   Alternative tags to RECT (optional if RECT/NORM_RECT is specified):
+//   WIDTH - The desired width of the output cropped image,
+//           based on image center
+//   HEIGHT - The desired height of the output cropped image,
+//            based on image center
+//
+// Output:
+//   One of the following two tags:
+//   IMAGE - Cropped ImageFrame
+//   IMAGE_GPU - Cropped GpuBuffer.
+//
+// Note: input_stream values take precedence over options defined in the graph.
+//
+namespace mediapipe {
+struct RectSpec {
+  int width;
+  int height;
+  int center_x;
+  int center_y;
+  float rotation;
+
+  bool operator==(const RectSpec& rect) const {
+    return (width == rect.width && height == rect.height &&
+            center_x == rect.center_x && center_y == rect.center_y &&
+            rotation == rect.rotation);
+  }
+};
+
+class ImageCroppingCalculator : public CalculatorBase {
+ public:
+  ImageCroppingCalculator() = default;
+  ~ImageCroppingCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+  static RectSpec GetCropSpecs(const CalculatorContext* cc, int src_width,
+                               int src_height);
+
+ private:
+  ::mediapipe::Status RenderCpu(CalculatorContext* cc);
+  ::mediapipe::Status RenderGpu(CalculatorContext* cc);
+  ::mediapipe::Status InitGpu(CalculatorContext* cc);
+  void GlRender();
+  void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
+                           int* dst_width, int* dst_height);
+
+  mediapipe::ImageCroppingCalculatorOptions options_;
+
+  bool use_gpu_ = false;
+  // Output texture corners (4) after transoformation in normalized coordinates.
+  float transformed_points_[8];
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  bool gpu_initialized_ = false;
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  GLuint program_ = 0;
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+};
+
+}  // namespace mediapipe
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_

mediapipe/calculators/image/image_cropping_calculator.proto

@@ -30,4 +30,14 @@ message ImageCroppingCalculatorOptions {
 
   // Rotation angle is counter-clockwise in radian.
   optional float rotation = 3 [default = 0.0];
+
+  // Normalized width and height of the output rect. Value is within [0, 1].
+  optional float norm_width = 4;
+  optional float norm_height = 5;
+
+  // Normalized location of the center of the output
+  // rectangle in image coordinates. Value is within [0, 1].
+  // The (0, 0) point is at the (top, left) corner.
+  optional float norm_center_x = 6 [default = 0];
+  optional float norm_center_y = 7 [default = 0];
 }

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -0,0 +1,216 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/image_cropping_calculator.h"
+
+#include <cmath>
+#include <memory>
+
+#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/tag_map.h"
+#include "mediapipe/framework/tool/tag_map_helper.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr int input_width = 100;
+constexpr int input_height = 100;
+
+constexpr char kRectTag[] = "RECT";
+constexpr char kHeightTag[] = "HEIGHT";
+constexpr char kWidthTag[] = "WIDTH";
+
+// Test normal case, where norm_width and norm_height in options are set.
+TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto cc =
+      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+                        tool::CreateTagMap({}).ValueOrDie());
+
+  RectSpec expectRect = {
+      .width = 60,
+      .height = 60,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when (width height) + (norm_width norm_height) are set in options.
+// width and height should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto cc =
+      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+                        tool::CreateTagMap({}).ValueOrDie());
+  RectSpec expectRect = {
+      .width = 50,
+      .height = 50,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when WIDTH HEIGHT are set from input stream,
+// and options has norm_width/height set.
+// WIDTH HEIGHT from input stream should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            input_stream: "WIDTH:crop_width"
+            input_stream: "HEIGHT:crop_height"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto inputTags = tool::CreateTagMap({
+                                          "HEIGHT:0:crop_height",
+                                          "WIDTH:0:crop_width",
+                                      })
+                       .ValueOrDie();
+  auto cc = CalculatorContext(&calculator_state, inputTags,
+                              tool::CreateTagMap({}).ValueOrDie());
+  auto& inputs = cc.Inputs();
+  inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
+  inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
+  RectSpec expectRect = {
+      .width = 1,
+      .height = 1,
+      .center_x = 50,
+      .center_y = 50,
+      .rotation = 0.3,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+// Test when RECT is set from input stream,
+// and options has norm_width/height set.
+// RECT from input stream should take precedence.
+TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
+  auto calculator_node =
+      ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
+          R"(
+            calculator: "ImageCroppingCalculator"
+            input_stream: "IMAGE_GPU:input_frames"
+            input_stream: "RECT:rect"
+            output_stream: "IMAGE_GPU:cropped_output_frames"
+            options: {
+              [mediapipe.ImageCroppingCalculatorOptions.ext] {
+                width: 50
+                height: 50
+                norm_width: 0.6
+                norm_height: 0.6
+                norm_center_x: 0.5
+                norm_center_y: 0.5
+                rotation: 0.3
+              }
+            }
+          )");
+
+  auto calculator_state =
+      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+  auto inputTags = tool::CreateTagMap({
+                                          "RECT:0:rect",
+                                      })
+                       .ValueOrDie();
+  auto cc = CalculatorContext(&calculator_state, inputTags,
+                              tool::CreateTagMap({}).ValueOrDie());
+  auto& inputs = cc.Inputs();
+  mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
+      R"(
+        width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
+      )");
+  inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
+  RectSpec expectRect = {
+      .width = 1,
+      .height = 1,
+      .center_x = 40,
+      .center_y = 40,
+      .rotation = 0.5,
+  };
+  EXPECT_EQ(
+      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+      expectRect);
+}  // TEST
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -138,8 +138,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Note: To enable horizontal or vertical flipping, specify them in the
 // calculator options. Flipping is applied after rotation.
 //
-// Note: Only scale mode STRETCH is currently supported on CPU,
-// and flipping is not yet supported either.
+// Note: Only scale mode STRETCH is currently supported on CPU.
 //
 class ImageTransformationCalculator : public CalculatorBase {
  public:
@@ -316,6 +315,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   cv::Mat input_mat = formats::MatView(&input_img);
   cv::Mat scaled_mat;
 
+  if (!output_height_ || !output_width_) {
+    output_height_ = input_height;
+    output_width_ = input_width;
+  }
+
   if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
     cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
   } else {
@@ -367,10 +371,21 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   cv::Mat rotation_mat = cv::getRotationMatrix2D(src_center, angle, 1.0);
   cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
 
+  cv::Mat flipped_mat;
+  if (options_.flip_horizontally() || options_.flip_vertically()) {
+    const int flip_code =
+        options_.flip_horizontally() && options_.flip_vertically()
+            ? -1
+            : options_.flip_horizontally();
+    cv::flip(rotated_mat, flipped_mat, flip_code);
+  } else {
+    flipped_mat = rotated_mat;
+  }
+
   std::unique_ptr<ImageFrame> output_frame(
       new ImageFrame(input_img.Format(), output_width, output_height));
   cv::Mat output_mat = formats::MatView(output_frame.get());
-  rotated_mat.copyTo(output_mat);
+  flipped_mat.copyTo(output_mat);
   cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
@@ -440,9 +455,8 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
   }
 
-  static mediapipe::FrameScaleMode scale_mode =
-      mediapipe::FrameScaleModeFromProto(scale_mode_,
-                                         mediapipe::FrameScaleMode::kStretch);
+  mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
+      scale_mode_, mediapipe::FrameScaleMode::kStretch);
   mediapipe::FrameRotation rotation =
       mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
 

mediapipe/calculators/image/scale_image_calculator.cc

@@ -474,13 +474,20 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
           input_width_, "x", input_height_));
     }
     if (input_format_ != image_frame.Format()) {
+      std::string image_frame_format_desc, input_format_desc;
+#ifdef MEDIAPIPE_MOBILE
+      image_frame_format_desc = std::to_string(image_frame.Format());
+      input_format_desc = std::to_string(input_format_);
+#else
       const proto_ns::EnumDescriptor* desc = ImageFormat::Format_descriptor();
+      image_frame_format_desc =
+          desc->FindValueByNumber(image_frame.Format())->DebugString();
+      input_format_desc = desc->FindValueByNumber(input_format_)->DebugString();
+#endif  // MEDIAPIPE_MOBILE
       return tool::StatusFail(absl::StrCat(
           "If a header specifies a format, then image frames on "
           "the stream must have that format.  Actual format ",
-          desc->FindValueByNumber(image_frame.Format())->DebugString(),
-          " but expected ",
-          desc->FindValueByNumber(input_format_)->DebugString()));
+          image_frame_format_desc, " but expected ", input_format_desc));
     }
   }
   return ::mediapipe::OkStatus();

mediapipe/calculators/image/testdata/BUILD

@@ -21,6 +21,7 @@ filegroup(
         "dino.jpg",
         "dino_quality_50.jpg",
         "dino_quality_80.jpg",
+        "front_camera_pixel2.jpg",
     ],
     visibility = ["//visibility:public"],
 )

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -34,6 +34,7 @@ namespace mediapipe {
 
 const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 const char kImageTag[] = "IMAGE";
+const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
 const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
 const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
 const char kBBoxTag[] = "BBOX";
@@ -145,6 +146,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
         }
         cc->Inputs().Tag(tag).Set<std::vector<Detection>>();
       }
+      if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) {
+        cc->Inputs().Tag(tag).Set<std::vector<float>>();
+      }
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
         cc->Inputs().Tag(tag).Set<std::vector<float>>();
       }
@@ -264,7 +268,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
     if (options.output_only_if_all_present()) {
       ::mediapipe::Status status = VerifySequence();
       if (!status.ok()) {
-        cc->GetCounter(status.error_message())->Increment();
+        cc->GetCounter(status.ToString())->Increment();
         return status;
       }
     }
@@ -344,6 +348,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                              sequence_.get());
         }
       }
+      if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) &&
+          !cc->Inputs().Tag(tag).IsEmpty()) {
+        std::string key =
+            tag.substr(sizeof(kFloatContextFeaturePrefixTag) /
+                           sizeof(*kFloatContextFeaturePrefixTag) -
+                       1);
+        RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream());
+        mpms::SetContextFeatureFloats(
+            key, cc->Inputs().Tag(tag).Get<std::vector<float>>(),
+            sequence_.get());
+      }
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag) &&
           !cc->Inputs().Tag(tag).IsEmpty()) {
         std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) /

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -194,6 +194,38 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   }
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
+  SetUpCalculator(
+      {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
+      {}, false, true);
+  auto input_sequence = absl::make_unique<tf::SequenceExample>();
+
+  auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
+  runner_->MutableInputs()
+      ->Tag("FLOAT_CONTEXT_FEATURE_TEST")
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+  vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
+  runner_->MutableInputs()
+      ->Tag("FLOAT_CONTEXT_FEATURE_OTHER")
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence),
+              testing::ElementsAre(3, 3));
+  ASSERT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence),
+              testing::ElementsAre(4, 4));
+}
+
 TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
   tf::Features context;
   (*context.mutable_feature())["TEST"].mutable_bytes_list()->add_value("YES");

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -454,7 +454,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
 
     // RET_CHECK on the tf::Status object itself in order to print an
     // informative error message.
-    RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
 
     const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
     cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -109,7 +109,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
 
     RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
     const tf::Status tf_status = session->session->Create(graph_def);
-    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString();
 
     for (const auto& key_value : options.tag_to_tensor_names()) {
       session->tag_to_tensor_map[key_value.first] = key_value.second;
@@ -119,7 +119,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
           session->session->Run({}, {}, initialization_op_names, {});
       // RET_CHECK on the tf::Status object itself in order to print an
       // informative error message.
-      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
     cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc

@@ -109,7 +109,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
 
     RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
     const tf::Status tf_status = session->session->Create(graph_def);
-    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString();
 
     for (const auto& key_value : options.tag_to_tensor_names()) {
       session->tag_to_tensor_map[key_value.first] = key_value.second;
@@ -119,7 +119,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
           session->session->Run({}, {}, initialization_op_names, {});
       // RET_CHECK on the tf::Status object itself in order to print an
       // informative error message.
-      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
     output_side_packets->Tag("SESSION") = Adopt(session.release());

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -140,7 +140,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
     if (!status.ok()) {
       return ::mediapipe::Status(
           static_cast<::mediapipe::StatusCode>(status.code()),
-          status.error_message());
+          status.ToString());
     }
 
     auto session = absl::make_unique<TensorFlowSession>();

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -135,7 +135,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
     if (!status.ok()) {
       return ::mediapipe::Status(
           static_cast<::mediapipe::StatusCode>(status.code()),
-          status.error_message());
+          status.ToString());
     }
 
     auto session = absl::make_unique<TensorFlowSession>();

mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc

@@ -81,7 +81,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
   auto tf_status = tensorflow::Env::Default()->NewRandomAccessFile(
       cc->InputSidePackets().Tag(kTFRecordPath).Get<std::string>(), &file);
   RET_CHECK(tf_status.ok())
-      << "Failed to open tfrecord file: " << tf_status.error_message();
+      << "Failed to open tfrecord file: " << tf_status.ToString();
   tensorflow::io::RecordReader reader(file.get(),
                                       tensorflow::io::RecordReaderOptions());
   tensorflow::uint64 offset = 0;
@@ -94,7 +94,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
   while (current_idx <= target_idx) {
     tf_status = reader.ReadRecord(&offset, &example_str);
     RET_CHECK(tf_status.ok())
-        << "Failed to read tfrecord: " << tf_status.error_message();
+        << "Failed to read tfrecord: " << tf_status.ToString();
     if (current_idx == target_idx) {
       if (cc->OutputSidePackets().HasTag(kExampleTag)) {
         tensorflow::Example tf_example;

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -294,11 +294,15 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
       if (use_quantized_tensors_) {
         RET_CHECK(image_frame.Format() != mediapipe::ImageFormat::VEC32F1)
             << "Only 8-bit input images are supported for quantization.";
+        quant.type = kTfLiteAffineQuantization;
+        quant.params = nullptr;
         // Optional: Set 'quant' quantization params here if needed.
         interpreter_->SetTensorParametersReadWrite(0, kTfLiteUInt8, "",
                                                    {channels_preserved}, quant);
       } else {
-        // Default TfLiteQuantization used for no quantization.
+        // Initialize structure for no quantization.
+        quant.type = kTfLiteNoQuantization;
+        quant.params = nullptr;
         interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "",
                                                    {channels_preserved}, quant);
       }
@@ -422,40 +426,35 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 #elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
-  {
-    id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    command_buffer.label = @"TfLiteConverterCalculatorConvert";
-    id<MTLComputeCommandEncoder> compute_encoder =
-        [command_buffer computeCommandEncoder];
-    [compute_encoder setComputePipelineState:gpu_data_out_->pipeline_state];
-    [compute_encoder setTexture:src_texture atIndex:0];
-    [compute_encoder setBuffer:gpu_data_out_->buffer offset:0 atIndex:1];
-    MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
-    MTLSize threadgroups =
-        MTLSizeMake(NumGroups(input.width(), kWorkgroupSize),
-                    NumGroups(input.height(), kWorkgroupSize), 1);
-    [compute_encoder dispatchThreadgroups:threadgroups
-                    threadsPerThreadgroup:threads_per_group];
-    [compute_encoder endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
-  }
+  id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+
+  id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
+  command_buffer.label = @"TfLiteConverterCalculatorConvertAndBlit";
+  id<MTLComputeCommandEncoder> compute_encoder =
+      [command_buffer computeCommandEncoder];
+  [compute_encoder setComputePipelineState:gpu_data_out_->pipeline_state];
+  [compute_encoder setTexture:src_texture atIndex:0];
+  [compute_encoder setBuffer:gpu_data_out_->buffer offset:0 atIndex:1];
+  MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
+  MTLSize threadgroups =
+      MTLSizeMake(NumGroups(input.width(), kWorkgroupSize),
+                  NumGroups(input.height(), kWorkgroupSize), 1);
+  [compute_encoder dispatchThreadgroups:threadgroups
+                  threadsPerThreadgroup:threads_per_group];
+  [compute_encoder endEncoding];
 
   // Copy into outputs.
   // TODO Avoid this copy.
   auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
   output_tensors->resize(1);
-  {
-    id<MTLDevice> device = gpu_helper_.mtlDevice;
-    output_tensors->at(0) =
-        [device newBufferWithLength:gpu_data_out_->elements * sizeof(float)
-                            options:MTLResourceStorageModeShared];
-    [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0)
-                               from:gpu_data_out_->buffer
-                           blocking:true
-                      commandBuffer:[gpu_helper_ commandBuffer]];
-  }
+  id<MTLDevice> device = gpu_helper_.mtlDevice;
+  output_tensors->at(0) =
+      [device newBufferWithLength:gpu_data_out_->elements * sizeof(float)
+                          options:MTLResourceStorageModeShared];
+  [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0)
+                             from:gpu_data_out_->buffer
+                         blocking:false
+                    commandBuffer:command_buffer];
 
   cc->Outputs()
       .Tag("TENSORS_GPU")

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -56,6 +56,10 @@
 #endif  // ANDROID
 
 namespace {
+// Commonly used to compute the number of blocks to launch in a kernel.
+int NumGroups(const int size, const int group_size) {  // NOLINT
+  return (size + group_size - 1) / group_size;
+}
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
@@ -176,12 +180,13 @@ class TfLiteInferenceCalculator : public CalculatorBase {
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
-  std::unique_ptr<GPUData> gpu_data_in_;
+  std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
 #elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
-  std::unique_ptr<GPUData> gpu_data_in_;
+  std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  id<MTLComputePipelineState> fp32_to_fp16_program_;
   TFLBufferConvert* converter_from_BPHWC4_ = nil;
 #endif
 
@@ -308,22 +313,41 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
-    RET_CHECK_EQ(input_tensors.size(), 1);
+    RET_CHECK_GT(input_tensors.size(), 0);
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
           // Explicit copy input.
-          RET_CHECK_CALL(CopyBuffer(input_tensors[0], gpu_data_in_->buffer));
+          gpu_data_in_.resize(input_tensors.size());
+          for (int i = 0; i < input_tensors.size(); ++i) {
+            RET_CHECK_CALL(
+                CopyBuffer(input_tensors[i], gpu_data_in_[i]->buffer));
+          }
+
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
-    RET_CHECK_EQ(input_tensors.size(), 1);
-    // Explicit copy input.
-    [MPPMetalUtil blitMetalBufferTo:gpu_data_in_->buffer
-                               from:input_tensors[0]
-                           blocking:true
-                      commandBuffer:[gpu_helper_ commandBuffer]];
+    RET_CHECK_GT(input_tensors.size(), 0);
+    // Explicit copy input with conversion float 32 bits to 16 bits.
+    gpu_data_in_.resize(input_tensors.size());
+    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+    command_buffer.label = @"TfLiteInferenceCalculatorConvert";
+    id<MTLComputeCommandEncoder> compute_encoder =
+        [command_buffer computeCommandEncoder];
+    [compute_encoder setComputePipelineState:fp32_to_fp16_program_];
+    for (int i = 0; i < input_tensors.size(); ++i) {
+      [compute_encoder setBuffer:input_tensors[i] offset:0 atIndex:0];
+      [compute_encoder setBuffer:gpu_data_in_[i]->buffer offset:0 atIndex:1];
+      constexpr int kWorkgroupSize = 64;  // Block size for GPU shader.
+      MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, 1, 1);
+      const int threadgroups =
+          NumGroups(gpu_data_in_[i]->elements, kWorkgroupSize);
+      [compute_encoder dispatchThreadgroups:MTLSizeMake(threadgroups, 1, 1)
+                      threadsPerThreadgroup:threads_per_group];
+    }
+    [compute_encoder endEncoding];
+    [command_buffer commit];
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif
@@ -404,7 +428,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     }
     [convert_command endEncoding];
     [command_buffer commit];
-    [command_buffer waitUntilCompleted];
     cc->Outputs()
         .Tag("TENSORS_GPU")
         .Add(output_tensors.release(), cc->InputTimestamp());
@@ -432,7 +455,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
         TfLiteGpuDelegateDelete(delegate_);
-        gpu_data_in_.reset();
+        for (int i = 0; i < gpu_data_in_.size(); ++i) {
+          gpu_data_in_[i].reset();
+        }
         for (int i = 0; i < gpu_data_out_.size(); ++i) {
           gpu_data_out_[i].reset();
         }
@@ -440,7 +465,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       }));
 #elif defined(MEDIAPIPE_IOS)
       TFLGpuDelegateDelete(delegate_);
-      gpu_data_in_.reset();
+      for (int i = 0; i < gpu_data_in_.size(); ++i) {
+        gpu_data_in_[i].reset();
+      }
       for (int i = 0; i < gpu_data_out_.size(); ++i) {
         gpu_data_out_[i].reset();
       }
@@ -464,7 +491,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // Get model name.
   if (!options.model_path().empty()) {
-    auto model_path = options.model_path();
+    std::string model_path = options.model_path();
 
     ASSIGN_OR_RETURN(model_path_, mediapipe::PathToResourceAsFile(model_path));
   } else {
@@ -545,24 +572,24 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   if (gpu_input_) {
     // Get input image sizes.
-    gpu_data_in_ = absl::make_unique<GPUData>();
     const auto& input_indices = interpreter_->inputs();
-    RET_CHECK_EQ(input_indices.size(), 1);  // TODO accept > 1.
-    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
-    gpu_data_in_->elements = 1;
-    for (int d = 0; d < tensor->dims->size; ++d) {
-      gpu_data_in_->elements *= tensor->dims->data[d];
+    gpu_data_in_.resize(input_indices.size());
+    for (int i = 0; i < input_indices.size(); ++i) {
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+      gpu_data_in_[i] = absl::make_unique<GPUData>();
+      gpu_data_in_[i]->elements = 1;
+      for (int d = 0; d < tensor->dims->size; ++d) {
+        gpu_data_in_[i]->elements *= tensor->dims->data[d];
+      }
+      // Create and bind input buffer.
+      RET_CHECK_CALL(
+          ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+              gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
+      RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
+                       delegate_, gpu_data_in_[i]->buffer.id(),
+                       interpreter_->inputs()[i]),
+                   kTfLiteOk);
     }
-    CHECK_GE(tensor->dims->data[3], 1);
-    CHECK_LE(tensor->dims->data[3], 4);
-    CHECK_NE(tensor->dims->data[3], 2);
-    // Create and bind input buffer.
-    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
-        gpu_data_in_->elements, &gpu_data_in_->buffer));
-    RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
-                     delegate_, gpu_data_in_->buffer.id(),
-                     interpreter_->inputs()[0]),  // First tensor only
-                 kTfLiteOk);
   }
   if (gpu_output_) {
     // Get output image sizes.
@@ -594,41 +621,68 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif  // OpenGL
 
 #if defined(MEDIAPIPE_IOS)
+  const int kHalfSize = 2;  // sizeof(half)
   // Configure and create the delegate.
   TFLGpuDelegateOptions options;
-  options.allow_precision_loss = false;  // Must match converter, F=float/T=half
+  options.allow_precision_loss = true;
   options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
   if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   if (gpu_input_) {
     // Get input image sizes.
-    gpu_data_in_ = absl::make_unique<GPUData>();
     const auto& input_indices = interpreter_->inputs();
-    RET_CHECK_EQ(input_indices.size(), 1);
-    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
-    gpu_data_in_->elements = 1;
-    // On iOS GPU, input must be 4 channels, regardless of what model expects.
-    {
-      gpu_data_in_->elements *= tensor->dims->data[0];  // batch
-      gpu_data_in_->elements *= tensor->dims->data[1];  // height
-      gpu_data_in_->elements *= tensor->dims->data[2];  // width
-      gpu_data_in_->elements *= 4;                      // channels
+    gpu_data_in_.resize(input_indices.size());
+    for (int i = 0; i < input_indices.size(); ++i) {
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
+      gpu_data_in_[i] = absl::make_unique<GPUData>();
+      gpu_data_in_[i]->shape.b = tensor->dims->data[0];
+      gpu_data_in_[i]->shape.h = tensor->dims->data[1];
+      gpu_data_in_[i]->shape.w = tensor->dims->data[2];
+      // On iOS GPU, input must be 4 channels, regardless of what model expects.
+      gpu_data_in_[i]->shape.c = 4;
+      gpu_data_in_[i]->elements =
+          gpu_data_in_[i]->shape.b * gpu_data_in_[i]->shape.h *
+          gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c;
+      // Input to model can be RGBA only.
+      if (tensor->dims->data[3] != 4) {
+        LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
+      }
+      const std::string shader_source =
+          absl::Substitute(R"(#include <metal_stdlib>
+        using namespace metal;
+        kernel void convertKernel(device float4* const input_buffer [[buffer(0)]],
+                                  device half4* output_buffer [[buffer(1)]],
+                                  uint gid [[thread_position_in_grid]]) {
+          if (gid >= $0) return;
+          output_buffer[gid] = half4(input_buffer[gid]);
+        })",
+                           gpu_data_in_[i]->elements / 4);
+      NSString* library_source =
+          [NSString stringWithUTF8String:shader_source.c_str()];
+      NSError* error = nil;
+      id<MTLLibrary> library =
+          [device newLibraryWithSource:library_source options:nil error:&error];
+      RET_CHECK(library != nil) << "Couldn't create shader library "
+                                << [[error localizedDescription] UTF8String];
+      id<MTLFunction> kernel_func = nil;
+      kernel_func = [library newFunctionWithName:@"convertKernel"];
+      RET_CHECK(kernel_func != nil) << "Couldn't create kernel function.";
+      fp32_to_fp16_program_ =
+          [device newComputePipelineStateWithFunction:kernel_func error:&error];
+      RET_CHECK(fp32_to_fp16_program_ != nil)
+          << "Couldn't create pipeline state "
+          << [[error localizedDescription] UTF8String];
+
+      // Create and bind input buffer.
+      gpu_data_in_[i]->buffer =
+          [device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
+                              options:MTLResourceStorageModeShared];
+      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+      RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
+                       delegate_, input_indices[i], gpu_data_in_[i]->buffer),
+                   true);
     }
-    // Input to model can be RGBA only.
-    if (tensor->dims->data[3] != 4) {
-      LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
-    }
-    // Create and bind input buffer.
-    gpu_data_in_->buffer =
-        [device newBufferWithLength:gpu_data_in_->elements * sizeof(float)
-                            options:MTLResourceStorageModeShared];
-    RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
-    RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
-                     delegate_,
-                     input_indices[0],  // First tensor only
-                     gpu_data_in_->buffer),
-                 true);
   }
   if (gpu_output_) {
     // Get output image sizes.
@@ -669,15 +723,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     interpreter_->SetAllowBufferHandleOutput(true);
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
       gpu_data_out_[i]->buffer =
-          [device newBufferWithLength:gpu_data_out_[i]->elements * sizeof(float)
+          [device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
                               options:MTLResourceStorageModeShared];
       RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                        delegate_, output_indices[i], gpu_data_out_[i]->buffer),
                    true);
     }
+
     // Create converter for GPU output.
     converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
-                                                            isFloat16:false
+                                                            isFloat16:true
                                                       convertToPBHWC4:false];
     if (converter_from_BPHWC4_ == nil) {
       return mediapipe::InternalError(

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -472,11 +472,11 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
   // Copy inputs.
   [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_boxes_buffer
                              from:input_tensors[0]
-                         blocking:true
+                         blocking:false
                     commandBuffer:[gpu_helper_ commandBuffer]];
   [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_scores_buffer
                              from:input_tensors[1]
-                         blocking:true
+                         blocking:false
                     commandBuffer:[gpu_helper_ commandBuffer]];
   if (!anchors_init_) {
     if (side_packet_anchors_) {
@@ -491,48 +491,37 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
       RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors);
       [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_anchors_buffer
                                  from:input_tensors[2]
-                             blocking:true
+                             blocking:false
                         commandBuffer:[gpu_helper_ commandBuffer]];
     }
     anchors_init_ = true;
   }
 
   // Run shaders.
-  {
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    command_buffer.label = @"TfLiteDecodeBoxes";
-    id<MTLComputeCommandEncoder> decode_command =
-        [command_buffer computeCommandEncoder];
-    [decode_command setComputePipelineState:gpu_data_->decode_program];
-    [decode_command setBuffer:gpu_data_->decoded_boxes_buffer
-                       offset:0
-                      atIndex:0];
-    [decode_command setBuffer:gpu_data_->raw_boxes_buffer offset:0 atIndex:1];
-    [decode_command setBuffer:gpu_data_->raw_anchors_buffer offset:0 atIndex:2];
-    MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
-    MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
-    [decode_command dispatchThreadgroups:decode_threadgroups
-                   threadsPerThreadgroup:decode_threads_per_group];
-    [decode_command endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
-  }
-  {
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    command_buffer.label = @"TfLiteScoreBoxes";
-    id<MTLComputeCommandEncoder> score_command =
-        [command_buffer computeCommandEncoder];
-    [score_command setComputePipelineState:gpu_data_->score_program];
-    [score_command setBuffer:gpu_data_->scored_boxes_buffer offset:0 atIndex:0];
-    [score_command setBuffer:gpu_data_->raw_scores_buffer offset:0 atIndex:1];
-    MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
-    MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
-    [score_command dispatchThreadgroups:score_threadgroups
+  id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+  command_buffer.label = @"TfLiteDecodeAndScoreBoxes";
+  id<MTLComputeCommandEncoder> command_encoder =
+      [command_buffer computeCommandEncoder];
+  [command_encoder setComputePipelineState:gpu_data_->decode_program];
+  [command_encoder setBuffer:gpu_data_->decoded_boxes_buffer
+                      offset:0
+                     atIndex:0];
+  [command_encoder setBuffer:gpu_data_->raw_boxes_buffer offset:0 atIndex:1];
+  [command_encoder setBuffer:gpu_data_->raw_anchors_buffer offset:0 atIndex:2];
+  MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
+  MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
+  [command_encoder dispatchThreadgroups:decode_threadgroups
+                  threadsPerThreadgroup:decode_threads_per_group];
+
+  [command_encoder setComputePipelineState:gpu_data_->score_program];
+  [command_encoder setBuffer:gpu_data_->scored_boxes_buffer offset:0 atIndex:0];
+  [command_encoder setBuffer:gpu_data_->raw_scores_buffer offset:0 atIndex:1];
+  MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
+  MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
+  [command_encoder dispatchThreadgroups:score_threadgroups
                   threadsPerThreadgroup:score_threads_per_group];
-    [score_command endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
-  }
+  [command_encoder endEncoding];
+  [MPPMetalUtil commitCommandBufferAndWait:command_buffer];
 
   // Copy decoded boxes from GPU to CPU.
   std::vector<float> boxes(num_boxes_ * num_coords_);

mediapipe/calculators/util/BUILD

@@ -508,6 +508,17 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "timed_box_list_to_render_data_calculator_proto",
+    srcs = ["timed_box_list_to_render_data_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util:color_proto",
+        "//mediapipe/util:render_data_proto",
+    ],
+)
+
 proto_library(
     name = "labels_to_render_data_calculator_proto",
     srcs = ["labels_to_render_data_calculator.proto"],
@@ -651,6 +662,37 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_cc_proto_library(
+    name = "timed_box_list_to_render_data_calculator_cc_proto",
+    srcs = ["timed_box_list_to_render_data_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util:color_cc_proto",
+        "//mediapipe/util:render_data_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":timed_box_list_to_render_data_calculator_proto"],
+)
+
+cc_library(
+    name = "timed_box_list_to_render_data_calculator",
+    srcs = ["timed_box_list_to_render_data_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":timed_box_list_to_render_data_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util:color_cc_proto",
+        "//mediapipe/util:render_data_cc_proto",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:tracking_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "labels_to_render_data_calculator",
     srcs = ["labels_to_render_data_calculator.cc"],

mediapipe/calculators/util/collection_has_min_size_calculator.h

@@ -37,6 +37,8 @@ namespace mediapipe {
 //    }
 //   }
 // }
+// Optionally, uses a side packet to override `min_size` specified in the
+// calculator options.
 template <typename IterableT>
 class CollectionHasMinSizeCalculator : public CalculatorBase {
  public:
@@ -54,6 +56,10 @@ class CollectionHasMinSizeCalculator : public CalculatorBase {
     cc->Inputs().Tag("ITERABLE").Set<IterableT>();
     cc->Outputs().Index(0).Set<bool>();
 
+    // Optional input side packet that determines `min_size_`.
+    if (cc->InputSidePackets().NumEntries() > 0) {
+      cc->InputSidePackets().Index(0).Set<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 
@@ -62,6 +68,11 @@ class CollectionHasMinSizeCalculator : public CalculatorBase {
     min_size_ =
         cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
             .min_size();
+    // Override `min_size` if passed as side packet.
+    if (cc->InputSidePackets().NumEntries() > 0 &&
+        !cc->InputSidePackets().Index(0).IsEmpty()) {
+      min_size_ = cc->InputSidePackets().Index(0).Get<int>();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc

@@ -0,0 +1,147 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "mediapipe/calculators/util/timed_box_list_to_render_data_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/color.pb.h"
+#include "mediapipe/util/render_data.pb.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/tracking.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kTimedBoxListTag[] = "BOX_LIST";
+constexpr char kRenderDataTag[] = "RENDER_DATA";
+
+void AddTimedBoxProtoToRenderData(
+    const TimedBoxProto& box_proto,
+    const TimedBoxListToRenderDataCalculatorOptions& options,
+    RenderData* render_data) {
+  if (box_proto.has_quad() && box_proto.quad().vertices_size() > 0 &&
+      box_proto.quad().vertices_size() % 2 == 0) {
+    const int num_corners = box_proto.quad().vertices_size() / 2;
+    for (int i = 0; i < num_corners; ++i) {
+      const int next_corner = (i + 1) % num_corners;
+      auto* line_annotation = render_data->add_render_annotations();
+      line_annotation->mutable_color()->set_r(options.box_color().r());
+      line_annotation->mutable_color()->set_g(options.box_color().g());
+      line_annotation->mutable_color()->set_b(options.box_color().b());
+      line_annotation->set_thickness(options.thickness());
+      RenderAnnotation::Line* line = line_annotation->mutable_line();
+      line->set_normalized(true);
+      line->set_x_start(box_proto.quad().vertices(i * 2));
+      line->set_y_start(box_proto.quad().vertices(i * 2 + 1));
+      line->set_x_end(box_proto.quad().vertices(next_corner * 2));
+      line->set_y_end(box_proto.quad().vertices(next_corner * 2 + 1));
+    }
+  } else {
+    auto* rect_annotation = render_data->add_render_annotations();
+    rect_annotation->mutable_color()->set_r(options.box_color().r());
+    rect_annotation->mutable_color()->set_g(options.box_color().g());
+    rect_annotation->mutable_color()->set_b(options.box_color().b());
+    rect_annotation->set_thickness(options.thickness());
+    RenderAnnotation::Rectangle* rect = rect_annotation->mutable_rectangle();
+    rect->set_normalized(true);
+    rect->set_left(box_proto.left());
+    rect->set_right(box_proto.right());
+    rect->set_top(box_proto.top());
+    rect->set_bottom(box_proto.bottom());
+    rect->set_rotation(box_proto.rotation());
+  }
+}
+
+}  // namespace
+
+// A calculator that converts TimedBoxProtoList proto to RenderData proto for
+// visualization. If the input TimedBoxProto contains `quad` field, this
+// calculator will draw a quadrilateral based on it. Otherwise this calculator
+// will draw a rotated rectangle based on `top`, `bottom`, `left`, `right` and
+// `rotation` fields
+//
+// Example config:
+// node {
+//   calculator: "TimedBoxListToRenderDataCalculator"
+//   input_stream: "BOX_LIST:landmarks"
+//   output_stream: "RENDER_DATA:render_data"
+//   options {
+//     [TimedBoxListToRenderDataCalculatorOptions.ext] {
+//       box_color { r: 0 g: 255 b: 0 }
+//       thickness: 4.0
+//     }
+//   }
+// }
+class TimedBoxListToRenderDataCalculator : public CalculatorBase {
+ public:
+  TimedBoxListToRenderDataCalculator() {}
+  ~TimedBoxListToRenderDataCalculator() override {}
+  TimedBoxListToRenderDataCalculator(
+      const TimedBoxListToRenderDataCalculator&) = delete;
+  TimedBoxListToRenderDataCalculator& operator=(
+      const TimedBoxListToRenderDataCalculator&) = delete;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  TimedBoxListToRenderDataCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(TimedBoxListToRenderDataCalculator);
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag(kTimedBoxListTag)) {
+    cc->Inputs().Tag(kTimedBoxListTag).Set<TimedBoxProtoList>();
+  }
+  cc->Outputs().Tag(kRenderDataTag).Set<RenderData>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::Open(
+    CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+  options_ = cc->Options<TimedBoxListToRenderDataCalculatorOptions>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimedBoxListToRenderDataCalculator::Process(
+    CalculatorContext* cc) {
+  auto render_data = absl::make_unique<RenderData>();
+
+  if (cc->Inputs().HasTag(kTimedBoxListTag)) {
+    const auto& box_list =
+        cc->Inputs().Tag(kTimedBoxListTag).Get<TimedBoxProtoList>();
+
+    for (const auto& box : box_list.box()) {
+      AddTimedBoxProtoToRenderData(box, options_, render_data.get());
+    }
+  }
+
+  cc->Outputs()
+      .Tag(kRenderDataTag)
+      .Add(render_data.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.proto

@@ -0,0 +1,32 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/color.proto";
+
+message TimedBoxListToRenderDataCalculatorOptions {
+  extend CalculatorOptions {
+    optional TimedBoxListToRenderDataCalculatorOptions ext = 289899854;
+  }
+
+  // Color of boxes.
+  optional Color box_color = 1;
+
+  // Thickness of the drawing of boxes.
+  optional double thickness = 2 [default = 1.0];
+}

mediapipe/calculators/video/BUILD

@@ -65,6 +65,15 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "video_pre_stream_calculator_proto",
+    srcs = ["video_pre_stream_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 mediapipe_cc_proto_library(
     name = "motion_analysis_calculator_cc_proto",
     srcs = ["motion_analysis_calculator.proto"],
@@ -98,6 +107,16 @@ mediapipe_cc_proto_library(
     deps = [":box_tracker_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "video_pre_stream_calculator_cc_proto",
+    srcs = ["video_pre_stream_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":video_pre_stream_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "flow_to_image_calculator_cc_proto",
     srcs = ["flow_to_image_calculator.proto"],
@@ -280,6 +299,19 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "video_pre_stream_calculator",
+    srcs = ["video_pre_stream_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":video_pre_stream_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:video_stream_header",
+    ],
+    alwayslink = 1,
+)
+
 filegroup(
     name = "test_videos",
     srcs = [
@@ -411,3 +443,17 @@ cc_test(
         "//mediapipe/util/tracking:tracking_cc_proto",
     ],
 )
+
+cc_test(
+    name = "video_pre_stream_calculator_test",
+    srcs = ["video_pre_stream_calculator_test.cc"],
+    deps = [
+        ":video_pre_stream_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)

mediapipe/calculators/video/opencv_video_decoder_calculator.cc

@@ -72,6 +72,8 @@ ImageFormat::Format GetImageFormat(int num_channels) {
 // OpenCV's VideoCapture doesn't decode audio tracks. If the audio tracks need
 // to be saved, specify an output side packet with tag "SAVED_AUDIO_PATH".
 // The calculator will call FFmpeg binary to save audio tracks as an aac file.
+// If the audio tracks can't be extracted by FFmpeg, the output side packet
+// will contain an empty std::string.
 //
 // Example config:
 // node {
@@ -150,13 +152,23 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
     if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) {
 #ifdef HAVE_FFMPEG
       std::string saved_audio_path = std::tmpnam(nullptr);
-      system(absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
-                          " -vn -f adts ", saved_audio_path)
-                 .c_str());
-      cc->OutputSidePackets()
-          .Tag("SAVED_AUDIO_PATH")
-          .Set(MakePacket<std::string>(saved_audio_path));
-
+      std::string ffmpeg_command =
+          absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path,
+                       " -vn -f adts ", saved_audio_path);
+      system(ffmpeg_command.c_str());
+      int status_code = system(absl::StrCat("ls ", saved_audio_path).c_str());
+      if (status_code == 0) {
+        cc->OutputSidePackets()
+            .Tag("SAVED_AUDIO_PATH")
+            .Set(MakePacket<std::string>(saved_audio_path));
+      } else {
+        LOG(WARNING) << "FFmpeg can't extract audio from " << input_file_path
+                     << " by executing the following command: "
+                     << ffmpeg_command;
+        cc->OutputSidePackets()
+            .Tag("SAVED_AUDIO_PATH")
+            .Set(MakePacket<std::string>(std::string()));
+      }
 #else
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
              << "OpenCVVideoDecoderCalculator can't save the audio file "

mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc

@@ -55,8 +55,12 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
   EXPECT_EQ(640, header.height);
   EXPECT_FLOAT_EQ(6.0f, header.duration);
   EXPECT_FLOAT_EQ(30.0f, header.frame_rate);
-  EXPECT_EQ(180, runner.Outputs().Tag("VIDEO").packets.size());
-  for (int i = 0; i < 180; ++i) {
+  // The number of the output packets should be 180.
+  // Some OpenCV version returns the first two frames with the same timestamp on
+  // macos and we might miss one frame here.
+  int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size();
+  EXPECT_GE(num_of_packets, 179);
+  for (int i = 0; i < num_of_packets; ++i) {
     Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i];
     cv::Mat output_mat =
         formats::MatView(&(image_frame_packet.Get<ImageFrame>()));
@@ -141,8 +145,12 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMkvVp8Video) {
   EXPECT_EQ(320, header.height);
   EXPECT_FLOAT_EQ(6.0f, header.duration);
   EXPECT_FLOAT_EQ(30.0f, header.frame_rate);
-  EXPECT_EQ(180, runner.Outputs().Tag("VIDEO").packets.size());
-  for (int i = 0; i < 180; ++i) {
+  // The number of the output packets should be 180.
+  // Some OpenCV version returns the first two frames with the same timestamp on
+  // macos and we might miss one frame here.
+  int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size();
+  EXPECT_GE(num_of_packets, 179);
+  for (int i = 0; i < num_of_packets; ++i) {
     Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i];
     cv::Mat output_mat =
         formats::MatView(&(image_frame_packet.Get<ImageFrame>()));

mediapipe/calculators/video/opencv_video_encoder_calculator.cc

@@ -183,14 +183,20 @@ class OpenCvVideoEncoderCalculator : public CalculatorBase {
 #ifdef HAVE_FFMPEG
     const std::string& audio_file_path =
         cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Get<std::string>();
-    // A temp output file is needed because FFmpeg can't do in-place editing.
-    const std::string temp_file_path = std::tmpnam(nullptr);
-    system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
-                        "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
-                        " -i ", audio_file_path,
-                        "  -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
-                        "&& rm ", temp_file_path)
-               .c_str());
+    if (audio_file_path.empty()) {
+      LOG(WARNING) << "OpenCvVideoEncoderCalculator isn't able to attach the "
+                      "audio tracks to the generated video because the audio "
+                      "file path is not specified.";
+    } else {
+      // A temp output file is needed because FFmpeg can't do in-place editing.
+      const std::string temp_file_path = std::tmpnam(nullptr);
+      system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path,
+                          "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path,
+                          " -i ", audio_file_path,
+                          "  -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_,
+                          "&& rm ", temp_file_path)
+                 .c_str());
+    }
 
 #else
     return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)

mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc

@@ -210,8 +210,8 @@ TEST(OpenCvVideoEncoderCalculatorTest, TestMkvVp8Video) {
   EXPECT_EQ(video_header.frame_rate,
             static_cast<double>(cap.get(cv::CAP_PROP_FPS)));
   EXPECT_EQ(video_header.duration,
-            static_cast<int>(cap.get(cv::CAP_PROP_FRAME_COUNT) /
-                             cap.get(cv::CAP_PROP_FPS)));
+            static_cast<int>(std::round(cap.get(cv::CAP_PROP_FRAME_COUNT) /
+                                        cap.get(cv::CAP_PROP_FPS))));
 }
 
 }  // namespace

mediapipe/calculators/video/tvl1_optical_flow_calculator.cc

@@ -88,7 +88,8 @@ class Tvl1OpticalFlowCalculator : public CalculatorBase {
   // cv::DenseOpticalFlow is not thread-safe. Invoking multiple
   // DenseOpticalFlow::calc() in parallel may lead to memory corruption or
   // memory leak.
-  std::list<cv::Ptr<cv::DenseOpticalFlow>> tvl1_computers_ GUARDED_BY(mutex_);
+  std::list<cv::Ptr<cv::DenseOpticalFlow>> tvl1_computers_
+      ABSL_GUARDED_BY(mutex_);
   absl::Mutex mutex_;
 };
 

mediapipe/calculators/video/video_pre_stream_calculator.cc

@@ -0,0 +1,142 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/video/video_pre_stream_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+
+namespace mediapipe {
+
+// Sets up VideoHeader based on the 1st ImageFrame and emits it with timestamp
+// PreStream. Note that this calculator only fills in format, width, and height,
+// i.e. frame_rate and duration will not be filled, unless:
+// 1) an existing VideoHeader is provided at PreStream(). In such case, the
+//    frame_rate and duration, if they exist, will be copied from the existing
+//    VideoHeader.
+// 2) you specify frame_rate and duration through the options. In this case, the
+//    options will overwrite the existing VideoHeader if it is available.
+//
+// Example config:
+// node {
+//   calculator: "VideoPreStreamCalculator"
+//   input_stream: "FRAME:cropped_frames"
+//   input_stream: "VIDEO_PRESTREAM:original_video_header"
+//   output_stream: "cropped_frames_video_header"
+// }
+//
+// or
+//
+// node {
+//   calculator: "VideoPreStreamCalculator"
+//   input_stream: "cropped_frames"
+//   output_stream: "video_header"
+// }
+class VideoPreStreamCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::Status ProcessWithFrameRateInPreStream(CalculatorContext* cc);
+  ::mediapipe::Status ProcessWithFrameRateInOptions(CalculatorContext* cc);
+
+  std::unique_ptr<VideoHeader> header_;
+  bool frame_rate_in_prestream_ = false;
+  bool emitted_ = false;
+};
+
+REGISTER_CALCULATOR(VideoPreStreamCalculator);
+
+::mediapipe::Status VideoPreStreamCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (!cc->Inputs().UsesTags()) {
+    cc->Inputs().Index(0).Set<ImageFrame>();
+  } else {
+    cc->Inputs().Tag("FRAME").Set<ImageFrame>();
+    cc->Inputs().Tag("VIDEO_PRESTREAM").Set<VideoHeader>();
+  }
+  cc->Outputs().Index(0).Set<VideoHeader>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoPreStreamCalculator::Open(CalculatorContext* cc) {
+  frame_rate_in_prestream_ = cc->Inputs().UsesTags() &&
+                             cc->Inputs().HasTag("FRAME") &&
+                             cc->Inputs().HasTag("VIDEO_PRESTREAM");
+  header_ = absl::make_unique<VideoHeader>();
+  return ::mediapipe::OkStatus();
+}
+::mediapipe::Status VideoPreStreamCalculator::ProcessWithFrameRateInPreStream(
+    CalculatorContext* cc) {
+  cc->GetCounter("ProcessWithFrameRateInPreStream")->Increment();
+  if (cc->InputTimestamp() == Timestamp::PreStream()) {
+    RET_CHECK(cc->Inputs().Tag("FRAME").IsEmpty());
+    RET_CHECK(!cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty());
+    *header_ = cc->Inputs().Tag("VIDEO_PRESTREAM").Get<VideoHeader>();
+    RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+  } else {
+    RET_CHECK(cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty())
+        << "Packet on VIDEO_PRESTREAM must come in at Timestamp::PreStream().";
+    RET_CHECK(!cc->Inputs().Tag("FRAME").IsEmpty());
+    const auto& frame = cc->Inputs().Tag("FRAME").Get<ImageFrame>();
+    header_->format = frame.Format();
+    header_->width = frame.Width();
+    header_->height = frame.Height();
+    RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+    cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream());
+    emitted_ = true;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VideoPreStreamCalculator::Process(CalculatorContext* cc) {
+  cc->GetCounter("Process")->Increment();
+  if (emitted_) {
+    return ::mediapipe::OkStatus();
+  }
+  if (frame_rate_in_prestream_) {
+    return ProcessWithFrameRateInPreStream(cc);
+  } else {
+    return ProcessWithFrameRateInOptions(cc);
+  }
+}
+
+::mediapipe::Status VideoPreStreamCalculator::ProcessWithFrameRateInOptions(
+    CalculatorContext* cc) {
+  cc->GetCounter("ProcessWithFrameRateInOptions")->Increment();
+  RET_CHECK_NE(cc->InputTimestamp(), Timestamp::PreStream());
+  const auto& frame = cc->Inputs().Index(0).Get<ImageFrame>();
+  header_->format = frame.Format();
+  header_->width = frame.Width();
+  header_->height = frame.Height();
+  const auto& options = cc->Options<VideoPreStreamCalculatorOptions>();
+  if (options.fps().has_value()) {
+    header_->frame_rate = options.fps().value();
+  } else if (options.fps().has_ratio()) {
+    const VideoPreStreamCalculatorOptions::Fps::Rational32& ratio =
+        options.fps().ratio();
+    if (ratio.numerator() > 0 && ratio.denominator() > 0) {
+      header_->frame_rate =
+          static_cast<double>(ratio.numerator()) / ratio.denominator();
+    }
+  }
+  RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero";
+  cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream());
+  emitted_ = true;
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/video_pre_stream_calculator.proto

@@ -0,0 +1,43 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message VideoPreStreamCalculatorOptions {
+  extend CalculatorOptions {
+    optional VideoPreStreamCalculatorOptions ext = 151386123;
+  }
+
+  // An arbitrary number of frames per second.
+  // Prefer the StandardFps enum to store industry-standard, safe FPS values.
+  message Fps {
+    // The possibly approximated value of the frame rate, in frames per second.
+    // Unsafe to use in accurate computations because prone to rounding errors.
+    // For example, the 23.976 FPS value has no exact representation as a
+    // double.
+    optional double value = 1;
+
+    message Rational32 {
+      optional int32 numerator = 1;
+      optional int32 denominator = 2;
+    }
+    // The exact value of the frame rate, as a rational number.
+    optional Rational32 ratio = 2;
+  }
+  optional Fps fps = 1;
+}

mediapipe/calculators/video/video_pre_stream_calculator_test.cc

@@ -0,0 +1,186 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "input"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "input"
+      output_stream: "output"
+      options {
+        [mediapipe.VideoPreStreamCalculatorOptions.ext] { fps { value: 3 } }
+      }
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto poller_status = graph.AddOutputStreamPoller("output");
+  MP_ASSERT_OK(poller_status.status());
+  OutputStreamPoller& poller = poller_status.ValueOrDie();
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+
+  // It is *not* VideoPreStreamCalculator's job to detect errors in an
+  // ImageFrame stream.  It just waits for the 1st ImageFrame, extracts info for
+  // VideoHeader, and emits it.  Thus, the following is fine.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input",
+      Adopt(new ImageFrame(ImageFormat::SRGBA, 3, 4)).At(Timestamp(1))));
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  const auto& video_header = packet.Get<VideoHeader>();
+  EXPECT_EQ(video_header.format, ImageFormat::SRGB);
+  EXPECT_EQ(video_header.width, 1);
+  EXPECT_EQ(video_header.height, 2);
+  EXPECT_EQ(video_header.frame_rate, 3);
+  EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream());
+  ASSERT_FALSE(poller.Next(&packet));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+}
+
+TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto poller_status = graph.AddOutputStreamPoller("output_header");
+  MP_ASSERT_OK(poller_status.status());
+  OutputStreamPoller& poller = poller_status.ValueOrDie();
+  MP_ASSERT_OK(graph.StartRun({}));
+  auto input_header = absl::make_unique<VideoHeader>();
+  input_header->frame_rate = 3.0;
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_header",
+      Adopt(input_header.release()).At(Timestamp::PreStream())));
+  MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  const auto& output_header = packet.Get<VideoHeader>();
+  EXPECT_EQ(output_header.format, ImageFormat::SRGB);
+  EXPECT_EQ(output_header.width, 1);
+  EXPECT_EQ(output_header.height, 2);
+  EXPECT_EQ(output_header.frame_rate, 3.0);
+  EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream());
+  ASSERT_FALSE(poller.Next(&packet));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+}
+
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInOptions) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "frame"
+      output_stream: "output_header"
+    })");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  ::mediapipe::Status status = graph.WaitUntilDone();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr("frame rate should be non-zero"));
+}
+
+// Input header missing.
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    }
+  )");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "frame",
+      Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("frame"));
+  MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+  ::mediapipe::Status status = graph.WaitUntilDone();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr("frame rate should be non-zero"));
+}
+
+// Input header not at prestream (before, with, and after frame data).
+TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream2) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "frame"
+    input_stream: "input_header"
+    node {
+      calculator: "VideoPreStreamCalculator"
+      input_stream: "FRAME:frame"
+      input_stream: "VIDEO_PRESTREAM:input_header"
+      output_stream: "output_header"
+    }
+  )");
+
+  for (int64 timestamp = -1; timestamp < 2; ++timestamp) {
+    CalculatorGraph graph;
+    MP_ASSERT_OK(graph.Initialize(config));
+    MP_ASSERT_OK(graph.StartRun({}));
+    auto input_header = absl::make_unique<VideoHeader>();
+    input_header->frame_rate = 3.0;
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input_header",
+        Adopt(input_header.release()).At(Timestamp(timestamp))));
+    MP_ASSERT_OK(graph.CloseInputStream("input_header"));
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "frame",
+        Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0))));
+    MP_ASSERT_OK(graph.CloseInputStream("frame"));
+    ::mediapipe::Status status = graph.WaitUntilDone();
+    EXPECT_FALSE(status.ok());
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/docs/autoflip.md

@@ -0,0 +1,341 @@
+# Saliency-Aware Video Cropping using AutoFlip
+
+## Introduction
+
+AutoFlip is an automatic video cropping pipeline built on top of MediaPipe. This
+example focuses on demonstrating how to use AutoFlip to convert an input video
+to arbitrary aspect ratios.
+
+For overall context on AutoFlip, please read this
+[Google AI Blog](https://mediapipe.page.link/autoflip).
+
+![graph is_required](images/autoflip_edited_example.gif)
+
+## Building
+
+Run the following command to build the AutoFlip pipeline:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/autoflip:run_autoflip
+```
+
+## Running
+
+```bash
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \
+  --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \
+  --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/absolute/path/to/save/the/output/video/file,aspect_ratio=1:1
+```
+
+Use the `aspect_ratio` flag to provide the output aspect ratio. The format
+should be `width:height`, where the `width` and `height` are two positive
+integers. AutoFlip supports both landscape-to-portrait and portrait-to-landscape
+conversions. The pipeline internally compares the target aspect ratio against
+the original one, and determines the correct conversion automatically.
+
+We have put a couple test videos under this
+[Google Drive folder](https://drive.google.com/corp/drive/u/0/folders/1KK9LV--Ey0UEVpxssVLhVl7dypgJSQgk).
+You could download the videos into your local file system, then modify the
+command above accordingly to run AutoFlip against the videos.
+
+## MediaPipe Graph
+
+![graph visualization](images/autoflip_graph.png)
+
+To visualize the graph as shown above, copy the text specification of the graph
+below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev).
+
+```bash
+# Autoflip graph that only renders the final cropped video. For use with
+# end user applications.
+max_queue_size: -1
+
+# VIDEO_PREP: Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:video_raw"
+  output_stream: "VIDEO_PRESTREAM:video_header"
+  output_side_packet: "SAVED_AUDIO_PATH:audio_path"
+}
+
+# VIDEO_PREP: Scale the input video before feature extraction.
+node {
+  calculator: "ScaleImageCalculator"
+  input_stream: "FRAMES:video_raw"
+  input_stream: "VIDEO_HEADER:video_header"
+  output_stream: "FRAMES:video_frames_scaled"
+  node_options: {
+    [type.googleapis.com/mediapipe.ScaleImageCalculatorOptions]: {
+      preserve_aspect_ratio: true
+      output_format: SRGB
+      target_width: 480
+      algorithm: DEFAULT_WITHOUT_UPSCALE
+    }
+  }
+}
+
+# VIDEO_PREP: Create a low frame rate stream for feature extraction.
+node {
+  calculator: "PacketThinnerCalculator"
+  input_stream: "video_frames_scaled"
+  output_stream: "video_frames_scaled_downsampled"
+  node_options: {
+    [type.googleapis.com/mediapipe.PacketThinnerCalculatorOptions]: {
+      thinner_type: ASYNC
+      period: 200000
+    }
+  }
+}
+
+# DETECTION: find borders around the video and major background color.
+node {
+  calculator: "BorderDetectionCalculator"
+  input_stream: "VIDEO:video_raw"
+  output_stream: "DETECTED_BORDERS:borders"
+}
+
+# DETECTION: find shot/scene boundaries on the full frame rate stream.
+node {
+  calculator: "ShotBoundaryCalculator"
+  input_stream: "VIDEO:video_frames_scaled"
+  output_stream: "IS_SHOT_CHANGE:shot_change"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.ShotBoundaryCalculatorOptions] {
+      min_shot_span: 0.2
+      min_motion: 0.3
+      window_size: 15
+      min_shot_measure: 10
+      min_motion_with_shot_measure: 0.05
+    }
+  }
+}
+
+# DETECTION: find faces on the down sampled stream
+node {
+  calculator: "AutoFlipFaceDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:face_detections"
+}
+node {
+  calculator: "FaceToRegionCalculator"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  input_stream: "FACES:face_detections"
+  output_stream: "REGIONS:face_regions"
+}
+
+# DETECTION: find objects on the down sampled stream
+node {
+  calculator: "AutoFlipObjectDetectionSubgraph"
+  input_stream: "VIDEO:video_frames_scaled_downsampled"
+  output_stream: "DETECTIONS:object_detections"
+}
+node {
+  calculator: "LocalizationToRegionCalculator"
+  input_stream: "DETECTIONS:object_detections"
+  output_stream: "REGIONS:object_regions"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.LocalizationToRegionCalculatorOptions] {
+      output_all_signals: true
+    }
+  }
+}
+
+# SIGNAL FUSION: Combine detections (with weights) on each frame
+node {
+  calculator: "SignalFusingCalculator"
+  input_stream: "shot_change"
+  input_stream: "face_regions"
+  input_stream: "object_regions"
+  output_stream: "salient_regions"
+  options {
+    [type.googleapis.com/mediapipe.autoflip.SignalFusingCalculatorOptions] {
+      signal_settings {
+        type { standard: FACE_CORE_LANDMARKS }
+        min_score: 0.85
+        max_score: 0.9
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_ALL_LANDMARKS }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type { standard: FACE_FULL }
+        min_score: 0.8
+        max_score: 0.85
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: HUMAN }
+        min_score: 0.75
+        max_score: 0.8
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: PET }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: CAR }
+        min_score: 0.7
+        max_score: 0.75
+        is_required: false
+      }
+      signal_settings {
+        type: { standard: OBJECT }
+        min_score: 0.1
+        max_score: 0.2
+        is_required: false
+      }
+    }
+  }
+}
+
+# CROPPING: make decisions about how to crop each frame.
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  node_options: {
+    [type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.5
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+
+# ENCODING(required): encode the video stream for the final cropped output.
+node {
+  calculator: "VideoPreStreamCalculator"
+  # Fetch frame format and dimension from input frames.
+  input_stream: "FRAME:cropped_frames"
+  # Copying frame rate and duration from original video.
+  input_stream: "VIDEO_PRESTREAM:video_header"
+  output_stream: "output_frames_video_header"
+}
+
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:cropped_frames"
+  input_stream: "VIDEO_PRESTREAM:output_frames_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  input_side_packet: "AUDIO_FILE_PATH:audio_path"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}
+```
+
+## Advanced Parameters
+
+### Required vs. Best-Effort Saliency Features
+
+AutoFlip allows users to implement and specify custom features to be used in the
+camera trajectory computation. If the user would like to detect and preserve
+scenes of lions in a wildlife protection video, for example, they could
+implement and add a feature detection calculator for lions into the pipeline.
+Refer to `AutoFlipFaceDetectionSubgraph` and `FaceToRegionCalculator`, or
+`AutoFlipObjectDetectionSubgraph` and `LocalizationToRegionCalculator` for
+examples of how to create new feature detection calculators.
+
+After adding different feature signals into the graph, use the
+`SignalFusingCalculator` node to specify types and weights for different feature
+signals. For example, in the graph above, we specified a `face_region` and an
+`object_region` input streams, to represent face signals and agnostic object
+signals, respectively.
+
+The larger the weight, the more important the features will be considered when
+AutoFlip computes the camera trajectory. Use the `is_required` flag to mark a
+feature as a hard constraint, in which case the computed camera trajectory will
+try best to cover these feature types in the cropped videos. If for some reason
+the required features cannot be all covered (for example, when they are too
+spread out in the video), AutoFlip will apply a padding effect to cover as much
+salient content as possible. See an illustration below.
+
+![graph is_required](images/autoflip_is_required.gif)
+
+### Stable vs Tracking Camera Motion
+
+AutoFlip makes a decision on each scene whether to have the cropped viewpoint
+follow an object or if the crop should remain stable (centered on detected
+objects). The parameter `motion_stabilization_threshold_percent` value is used
+to make the decision to track action or keep the camera stable. If, over the
+duration of the scene, all detected focus objects remain within this ratio of
+the frame (e.g. 0.5 = 50% or 1920 * .5 = 960 pixels on 1080p video) then the
+camera is held steady. Otherwise the camera tracks activity within the frame.
+
+### Snap To Center
+
+For some scenes the camera viewpoint will remain stable at the center of
+activity (see `motion_stabilization_threshold_percent` setting). In this case,
+if the determined best stable viewpoint is within
+`snap_center_max_distance_percent` of the frame's center the camera will be
+shifted to be locked to the center of the frame. This setting is useful for
+videos where the camera operator did a good job already centering content or if
+titles and logos are expected to appear in the center of the frame. It may be
+less useful on raw content where objects are not already well positioned on
+screen.
+
+### Visualization to Facilitate Debugging
+
+`SceneCroppingCalculator` provides two extra output streams
+`KEY_FRAME_CROP_REGION_VIZ_FRAMES` and `SALIENT_POINT_FRAME_VIZ_FRAMES` to
+visualize the cropping window as well as salient points detected on each frame.
+You could modify the `SceneCroppingCalculator` node like below to enable these
+two output streams.
+
+```bash
+node {
+  calculator: "SceneCroppingCalculator"
+  input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio"
+  input_stream: "VIDEO_FRAMES:video_raw"
+  input_stream: "KEY_FRAMES:video_frames_scaled_downsampled"
+  input_stream: "DETECTION_FEATURES:salient_regions"
+  input_stream: "STATIC_FEATURES:borders"
+  input_stream: "SHOT_BOUNDARIES:shot_change"
+  output_stream: "CROPPED_FRAMES:cropped_frames"
+  output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames"
+  output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames"
+  node_options: {
+    [type.googleapis.com/mediapipe.autoflip.SceneCroppingCalculatorOptions]: {
+      max_scene_size: 600
+      key_frame_crop_options: {
+        score_aggregation_type: CONSTANT
+      }
+      scene_camera_motion_analyzer_options: {
+        motion_stabilization_threshold_percent: 0.5
+        salient_point_bound: 0.499
+      }
+      padding_parameters: {
+        blur_cv_size: 200
+        overlay_opacity: 0.6
+      }
+      target_size_type: MAXIMIZE_TARGET_DIMENSION
+    }
+  }
+}
+```

mediapipe/docs/calculator.md

@@ -151,7 +151,6 @@ node {
    input_stream: "room_video_tick_signal"
    output_stream: "cloned_room_mic_signal"
    output_stream: "cloned_lighting_sensor"
-   output_stream: "cloned_video_tick_signal"
  }
 ```
 

mediapipe/docs/examples.md

@@ -108,9 +108,9 @@ and model details are described in the
 [Hello World for C++](./hello_world_desktop.md) shows how to run a simple graph
 using the MediaPipe C++ APIs.
 
-### Feature Extration for YouTube-8M Challenge
+### Feature Extraction and Model Inference for YouTube-8M Challenge
 
-[Feature Extration and Model Inference for YouTube-8M Challenge](./youtube_8m.md)
+[Feature Extraction and Model Inference for YouTube-8M Challenge](./youtube_8m.md)
 shows how to use MediaPipe to prepare training data for the YouTube-8M Challenge
 and do the model inference with the baseline model.
 
@@ -120,6 +120,11 @@ and do the model inference with the baseline model.
 MediaPipe for media processing to prepare video data sets for training a
 TensorFlow model.
 
+### AutoFlip - Automatic video cropping
+
+[AutoFlip](./autoflip.md) shows how to use MediaPipe to build an automatic video
+cropping pipeline that can convert an input video to arbitrary aspect ratios.
+
 ### Object Detection on Desktop
 
 [Object Detection on Desktop](./object_detection_desktop.md) shows how to run
@@ -137,6 +142,7 @@ GPU with live video from a webcam.
 *   [Desktop GPU](./face_detection_desktop.md)
 *   [Desktop CPU](./face_detection_desktop.md)
 
+
 ### Hand Tracking on Desktop with Webcam
 
 [Hand Tracking on Desktop with Webcam](./hand_tracking_desktop.md) shows how to
@@ -179,3 +185,18 @@ EdgeTPU on
 [Face Detection on Coral with Webcam](./face_detection_coral_devboard.md) shows
 how to use quantized face detection TFlite model accelerated with EdgeTPU on
 [Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).
+
+
+## Web Browser
+
+Below are samples that can directly be run in your web browser.
+See more details in [MediaPipe on the Web](./web.md) and
+[Google Developer blog post](https://mediapipe.page.link/webdevblog)
+
+### [Face Detection In Browser](https://viz.mediapipe.dev/demo/face_detection)
+
+### [Hand Detection In Browser](https://viz.mediapipe.dev/demo/hand_detection)
+
+### [Hand Tracking In Browser](https://viz.mediapipe.dev/demo/hand_tracking)
+
+### [Hair Segmentation In Browser](https://viz.mediapipe.dev/demo/hair_segmentation)

mediapipe/docs/face_detection_coral_devboard.md

@@ -2,9 +2,10 @@
 
 MediaPipe is able to run cross platform across device types like desktop, mobile
 and edge devices. Here is an example of running MediaPipe
-[face detection pipeline](./face_detection_desktop.md) on edge device like
-[Google Coral dev board](https://coral.withgoogle.com/products/dev-board) with
-[Edge TPU](https://cloud.google.com/edge-tpu/). This MediaPipe Coral face
+[face detection pipeline](./face_detection_desktop.md) on edge device like the
+[Coral Dev Board](https://coral.ai/products/dev-board).
+
+This MediaPipe Coral face
 detection pipeline is running [coral specific quantized version](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
 of the [MediaPipe face detection TFLite model](https://github.com/google/mediapipe/blob/master/mediapipe/models/face_detection_front.tflite)
 accelerated on Edge TPU.
@@ -13,8 +14,10 @@ accelerated on Edge TPU.
 
 We recommend building the MediaPipe binaries not on the edge device due to
 limited compute resulting in long build times. Instead, we will build MediaPipe
-binaries using Docker containers on a more powerful host machine. For step by
-step details of cross compiling and running MediaPipe binaries on Coral dev
-board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/README.md).
+binaries using Docker containers on a more powerful host machine.
 
-![Face Detection running on Coral](images/face_detection_demo_coral.jpg)
+For step by
+step details of cross compiling and running MediaPipe binaries on the Coral Dev
+Board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral).
+
+![Face Detection running on Coral](images/face_detection_demo_coral.gif)

mediapipe/docs/face_detection_desktop.md

@@ -18,7 +18,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Face Detection Demo with Webcam (CPU)
 
@@ -66,6 +68,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_detection/face_de
     --calculator_graph_config_file=mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/face_detection_desktop.png)

mediapipe/docs/framework_concepts.md

@@ -24,9 +24,9 @@ Mediapipe. At a minimum, a new calculator must implement the below four methods
 * `GetContract()`
   * Calculator authors can specify the expected types of inputs and outputs of a calculator in GetContract(). When a graph is initialized, the framework calls a static method to verify if the packet types of the connected inputs and outputs match the information in this specification.
 * `Open()`
-  * After a graph starts, the framework calls `Open()`. The input side packets are available to the calculator at this point. `Open()` interprets the node configuration (see Section \ref{graph_config}) operations and prepares the calculator's per-graph-run state. This function may also write packets to calculator outputs. An error during `Open()` can terminate the graph run.
+  * After a graph starts, the framework calls `Open()`. The input side packets are available to the calculator at this point. `Open()` interprets the node configuration operations (see Section [GraphConfig](#graphconfig)) and prepares the calculator's per-graph-run state. This function may also write packets to calculator outputs. An error during `Open()` can terminate the graph run.
 * `Process()`
-  * For a calculator with inputs, the framework calls `Process()` repeatedly whenever at least one input stream has a packet available. The framework by default guarantees that all inputs have the same timestamp (see Section \ref{scheduling} for more information). Multiple `Process()` calls can be invoked simultaneously when parallel execution is enabled. If an error occurs during `Process()`, the framework calls `Close()` and the graph run terminates.
+  * For a calculator with inputs, the framework calls `Process()` repeatedly whenever at least one input stream has a packet available. The framework by default guarantees that all inputs have the same timestamp (see [Framework Architecture](scheduling_sync.md) for more information). Multiple `Process()` calls can be invoked simultaneously when parallel execution is enabled. If an error occurs during `Process()`, the framework calls `Close()` and the graph run terminates.
 * `Close()`
   * After all calls to `Process()` finish or when all input streams close, the framework calls `Close()`. This function is always called if `Open()` was called and succeeded and even if the graph run terminated because of an error. No inputs are available via any input streams during `Close()`, but it still has access to input side packets and therefore may write outputs. After `Close()` returns, the calculator should be considered a dead node. The calculator object is destroyed as soon as the graph finishes running.
 

mediapipe/docs/gpu.md

@@ -2,6 +2,7 @@
 
 -   [Overview](#overview)
 -   [OpenGL Support](#opengl-support)
+-   [Desktop GPUs](#desktop-gpu-linux)
 -   [Life of a GPU calculator](#life-of-a-gpu-calculator)
 -   [GpuBuffer to ImageFrame converters](#gpubuffer-to-imageframe-converters)
 -   [Disable GPU support](#disable-gpu-support)
@@ -22,13 +23,60 @@ Below are the design principles for GPU support in MediaPipe
    * Because different platforms may require different techniques for best performance, the API should allow flexibility in the way things are implemented behind the scenes.
    * A calculator should be allowed maximum flexibility in using the GPU for all or part of its operation, combining it with the CPU if necessary.
 
-### OpenGL support
+### OpenGL Support
 
-MediaPipe supports OpenGL ES up to version 3.2 on Android and up to ES 3.0 on iOS. In addition, MediaPipe also supports Metal on iOS.
+MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
+on iOS. In addition, MediaPipe also supports Metal on iOS.
 
-   * MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this can be very useful in graphs that combine a slower GPU inference path (eg, at 10 FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context corresponds to one sequential command queue, using the same context for both tasks would reduce the rendering frame rate. One challenge MediaPipe's use of multiple contexts solves is the ability to communicate across them. An example scenario is one with an input video that is sent to both the rendering and inferences paths, and rendering needs to have access to the latest output from inference.
+OpenGL ES 3.1 or greater is required (on Android/Linux systems) for running
+machine learning inference calculators and graphs.
 
-   * An OpenGL context cannot be accessed by multiple threads at the same time. Furthermore, switching the active GL context on the same thread can be slow on some Android devices. Therefore, our approach is to have one dedicated thread per context. Each thread issues GL commands, building up a serial command queue on its context, which is then executed by the GPU asynchronously.
+MediaPipe allows graphs to run OpenGL in multiple GL contexts. For example, this
+can be very useful in graphs that combine a slower GPU inference path (eg, at 10
+FPS) with a faster GPU rendering path (eg, at 30 FPS): since one GL context
+corresponds to one sequential command queue, using the same context for both
+tasks would reduce the rendering frame rate.
+
+One challenge MediaPipe's use of multiple contexts solves is the ability to
+communicate across them. An example scenario is one with an input video that is
+sent to both the rendering and inferences paths, and rendering needs to have
+access to the latest output from inference.
+
+An OpenGL context cannot be accessed by multiple threads at the same time.
+Furthermore, switching the active GL context on the same thread can be slow on
+some Android devices. Therefore, our approach is to have one dedicated thread
+per context. Each thread issues GL commands, building up a serial command queue
+on its context, which is then executed by the GPU asynchronously.
+
+#### Desktop GPU (Linux)
+
+MediaPipe GPU can run on linux systems with video cards that support OpenGL ES
+3.1 and up.
+
+To check if your linux desktop GPU can run mediapipe:
+
+```bash
+$ sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
+$ sudo apt-get install mesa-utils
+$ glxinfo | grep -i opengl
+```
+
+My linux box prints:
+
+```bash
+$ glxinfo | grep -i opengl
+...
+OpenGL ES profile version string: OpenGL ES 3.2 NVIDIA 430.50
+OpenGL ES profile shading language version string: OpenGL ES GLSL ES 3.20
+OpenGL ES profile extensions:
+```
+
+*^notice the OpenGL ES 3.2 text^*
+
+To run MediaPipe GPU on desktop, you need to see ES 3.1 or greater printed.
+
+If OpenGL ES is not printed, or is below 3.1, then the GPU inference will not
+run.
 
 ### Life of a GPU calculator
 

mediapipe/docs/hair_segmentation_desktop.md

@@ -15,7 +15,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Hair Segmentation Demo with Webcam (GPU)
 
@@ -40,6 +42,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hair_segmentation/hair
     --calculator_graph_config_file=mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![hair_segmentation_mobile_gpu_graph](images/mobile/hair_segmentation_mobile_gpu.png)

mediapipe/docs/hand_tracking_desktop.md

@@ -17,7 +17,9 @@ Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
 see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
-Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Hand Tracking Demo with Webcam (CPU)
 
@@ -61,6 +63,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tra
     --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/hand_tracking_desktop.png)

mediapipe/docs/install.md

@@ -7,8 +7,9 @@ future.
 Note: If you plan to use TensorFlow calculators and example apps, there is a
 known issue with gcc and g++ version 6.3 and 7.3. Please use other versions.
 
-Note: To make Mediapipe work with TensorFlow, please install the python "future"
-library and the python "six" library using `pip install --user future six`.
+Note: To make Mediapipe work with TensorFlow, please set Python 3.7 as the
+default Python version and install the Python "six" library by running `pip3
+install --user six`.
 
 Choose your operating system:
 
@@ -39,19 +40,12 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (0.24.1 and above required).
+2.  Install Bazel (version between 0.24.1 and 1.2.1).
 
-    Option 1. Use package manager tool to install the latest version of Bazel.
-
-    ```bash
-    $ sudo apt-get install bazel
-
-    # Run 'bazel version' to check version of bazel installed
-    ```
-
-    Option 2. Follow the official
+    Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install any version of Bazel manually.
+    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    yet.
 
 3.  Install OpenCV and FFmpeg.
 
@@ -158,11 +152,12 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (0.24.1 and above required).
+2.  Install Bazel (version between 0.24.1 and 1.2.1).
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel manually.
+    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    yet.
 
 3.  Install OpenCV.
 
@@ -236,7 +231,7 @@ To build and run iOS apps:
 
     *   Install [Homebrew](https://brew.sh).
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
-        Tools by `xcode-select install`.
+        Tools by `xcode-select --install`.
 
 2.  Checkout MediaPipe repository.
 
@@ -271,12 +266,10 @@ To build and run iOS apps:
 
     ```bash
     $ brew install opencv@3
-    ```
 
-    Note: If you do `$brew install opencv`, there is a known issue caused by the
-    glog dependency of OpenCV 4.1.1 or above. The problem is solvable by
-    uninstalling the glog. You need to do `$ brew uninstall
-    --ignore-dependencies glog`
+    # There is a known issue caused by the glog dependency. Uninstall glog.
+    $ brew uninstall --ignore-dependencies glog
+    ```
 
     Option 2. Use MacPorts package manager tool to install the OpenCV libraries.
 
@@ -338,7 +331,17 @@ To build and run iOS apps:
 
     ```
 
-5.  Run the [Hello World desktop example](./hello_world_desktop.md).
+5.  Make sure that Python 3 and the Python "six" library are installed.
+
+    ```
+    $ brew install python
+    $ sudo ln -s -f /usr/local/bin/python3.7 /usr/local/bin/python
+    $ python --version
+    Python 3.7.4
+    $ pip3 install --user six
+    ```
+
+6.  Run the [Hello World desktop example](./hello_world_desktop.md).
 
     ```bash
     $ export GLOG_logtostderr=1
@@ -361,6 +364,9 @@ To build and run iOS apps:
 
 ### Installing on Windows Subsystem for Linux (WSL)
 
+Note: WSL has historically not provided access to USB cameras. Mediapipe can use
+a video file as input.
+
 1.  Follow the
     [instruction](https://docs.microsoft.com/en-us/windows/wsl/install-win10) to
     install Windows Sysystem for Linux (Ubuntu).
@@ -380,10 +386,10 @@ To build and run iOS apps:
 4.  Install the needed packages.
 
     ```bash
-    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y --no-install-recommends build-essential git python zip adb openjdk-8-jdk
+    username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel (0.24.1 and above required).
+5.  Install Bazel (version between 0.24.1 and 1.2.1).
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
@@ -568,6 +574,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
 
 Requirements:
 
+*   Java Runtime.
 *   Android SDK release 28.0.3 and above.
 *   Android NDK r17c and above.
 

mediapipe/docs/mediapipe_ios_setup.md

@@ -1,28 +1,32 @@
 ## Setting up MediaPipe for iOS
 
-1.  Install [Xcode](https://developer.apple.com/xcode/).
+1.  Install [Xcode](https://developer.apple.com/xcode/) and the Command Line
+    Tools.
 
-    Follow Apple's instructions to obtain the required developemnt certificates
-    and provisioning profiles for your iOS device.
+    Follow Apple's instructions to obtain the required development certificates
+    and provisioning profiles for your iOS device. Install the Command Line
+    Tools by
 
-2.  Install [Bazel](https://bazel.build/).
+    ```bash
+    xcode-select --install
+    ```
+
+2.  Install [Bazel 1.1.0](https://bazel.build/).
 
-    See their
-    [instructions](https://docs.bazel.build/versions/master/install-os-x.html).
     We recommend using [Homebrew](https://brew.sh/):
 
     ```bash
-    brew tap bazelbuild/tap
-    brew install bazelbuild/tap/bazel
+    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb
     ```
 
-3.  Install python "future" and "six".
+3.  Set Python 3.7 as the default Python version and install the Python "six"
+    library.
 
-    To make Mediapipe work with TensorFlow, please install the python "future"
-    library and the python "six" library:
+    To make Mediapipe work with TensorFlow, please set Python 3.7 as the default
+    Python version and install the Python "six" library.
 
     ```bash
-    pip install --user future six
+    pip3 install --user six
     ```
 
 4.  Clone the MediaPipe repository.

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -19,7 +19,8 @@ see
 [step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
 
 Note: If MediaPipe depends on OpenCV 2, please see the
-[known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+[known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
+section.
 
 ### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU)
 
@@ -61,6 +62,9 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/mu
     --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt
 ```
 
+Issues running? Please first
+[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
+
 #### Graph
 
 ![graph visualization](images/multi_hand_tracking_desktop.png)

mediapipe/docs/object_detection_coral_devboard.md

@@ -3,8 +3,9 @@
 MediaPipe is able to run cross platform across device types like desktop, mobile
 and edge devices. Here is an example of running MediaPipe
 [object detection pipeline](./object_detection_desktop.md) on edge device like
-[Google Coral dev board](https://coral.withgoogle.com/products/dev-board) with
-[Edge TPU](https://cloud.google.com/edge-tpu/). This MediaPipe Coral object
+the [Coral Dev Board](https://coral.ai/products/dev-board).
+
+This MediaPipe Coral object
 detection pipeline is running [coral specific quantized version](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
 of the [MediaPipe object detection TFLite model](https://github.com/google/mediapipe/blob/master/mediapipe/models/object_detection_front.tflite)
 accelerated on Edge TPU.
@@ -13,8 +14,10 @@ accelerated on Edge TPU.
 
 We recommend building the MediaPipe binaries not on the edge device due to
 limited compute resulting in long build times. Instead, we will build MediaPipe
-binaries using Docker containers on a more powerful host machine. For step by
-step details of cross compiling and running MediaPipe binaries on Coral dev
-board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/blob/master/mediapipe/examples/coral/README.md).
+binaries using Docker containers on a more powerful host machine.
+
+For step by
+step details of cross compiling and running MediaPipe binaries on the Coral Dev
+Board, please refer to [README.md in MediaPipe Coral example folder](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral).
 
 ![Object Detection running on Coral](images/object_detection_demo_coral.jpg)

mediapipe/docs/object_detection_desktop.md

@@ -214,7 +214,6 @@ To build and run the TensorFlow Lite example on desktop (CPU) with Webcam, run:
 # Video from webcam running on desktop CPU
 $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
     mediapipe/examples/desktop/object_detection:object_detection_cpu
-
 # It should print:
 #Target //mediapipe/examples/desktop/object_detection:object_detection_cpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_cpu

mediapipe/docs/object_tracking_mobile_gpu.md

@@ -5,8 +5,8 @@ This doc focuses on the
 that performs object detection and tracking.
 Note that object detection is using TensorFlow Lite on GPU while tracking is using CPU.
 
-For overall context on hand detection and hand tracking, please read this
-[Google Developer Blog](https://mediapipe.page.link/objecttrackingblog).
+For overall context on object detection and tracking, please read this
+[Google Developers Blog](https://mediapipe.page.link/objecttrackingblog).
 
 ![object_tracking_android_gpu_gif](images/mobile/object_tracking_android_gpu.gif)
 
@@ -238,7 +238,7 @@ tracking library that can be used for other use cases.
 
 ![object_tracking_subgraph](images/mobile/object_tracking_subgraph.png)
 
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_tracking.pbtxt)
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/object_tracking_gpu.pbtxt)
 
 ```bash
 # MediaPipe object tracking subgraph.
@@ -304,7 +304,7 @@ node: {
 
 ![box_tracking_subgraph](images/mobile/box_tracking_subgraph.png)
 
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking.pbtxt)
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/box_tracking_gpu.pbtxt)
 
 ```bash
 # MediaPipe box tracking subgraph.
@@ -437,7 +437,7 @@ node: {
 
 ### Renderer Subgraph
 
-![hand_renderer_gpu_subgraph.pbtxt](images/mobile/object_tracking_renderer_gpu_subgraph.png)
+![object_tracking_renderer_gpu_subgraph.pbtxt](images/mobile/object_tracking_renderer_gpu_subgraph.png)
 
 [Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt)
 
@@ -473,3 +473,19 @@ node {
 }
 
 ```
+
+## Desktop
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/object_tracking)
+
+Note that object detection is using TensorFlow Lite on CPU and tracking is using
+CPU.
+
+To build and run the app:
+
+```bash
+bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
+
+bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt
+```

mediapipe/docs/web.md

@@ -7,6 +7,8 @@ in the browser client-side. The official API is under construction, but the core
 technology has been proven effective, and we can already show interactive
 cross-platform demos using your live webcam.
 
+[For more details, read this Google Developer blog post](https://mediapipe.page.link/webdevblog)
+
 ![image](images/web_effect.gif) ![image](images/web_segmentation.gif)
 
 ### Hand Tracking (with and without SIMD support)
@@ -21,6 +23,3 @@ support.  Below are two different versions of the
   1. WebAssembly MVP [demo](https://mediapipe.page.link/cds-ht) running around 5-8 frames per second on Desktop Chrome
 
   2. WebAssembly SIMD [demo](https://mediapipe.page.link/cds-ht-simd) running around 15-18 frames per second on *Canary* Chrome for Desktop, which must additionally be launched with the option `--js-flags="--experimental-wasm-simd"`
-
-
-NOTE: This page is a work-in-progress. More to come soon!

mediapipe/docs/youtube_8m.md

@@ -1,4 +1,4 @@
-# Feature Extration and Model Inference for YouTube-8M Challenge
+# Feature Extraction and Model Inference for YouTube-8M Challenge
 
 MediaPipe is a useful and general framework for media processing that can assist
 with research, development, and deployment of ML models. This example focuses on
@@ -31,7 +31,9 @@ videos.
 
 ### Steps to run the YouTube-8M feature extraction graph
 
-1.  Checkout the mediapipe repository.
+1.  Checkout the repository and follow
+    [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md)
+    to set up MediaPipe.
 
     ```bash
     git clone https://github.com/google/mediapipe.git

mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java

@@ -149,12 +149,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD

@@ -61,7 +61,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java

@@ -180,12 +180,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -83,7 +83,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java

@@ -158,12 +158,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD

@@ -62,7 +62,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD

@@ -55,6 +55,7 @@ android_library(
     resource_files = glob(["res/**"]),
     deps = [
         ":mediapipe_jni_lib",
+        "//mediapipe/framework/formats:detection_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
         "//mediapipe/java/com/google/mediapipe/components:android_components",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",
@@ -62,7 +63,6 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:androidx_legacy_support_v4",
-        "//third_party:androidx_material",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
         "@androidx_concurrent_futures//jar",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java

@@ -17,18 +17,22 @@ package com.google.mediapipe.apps.objectdetectiongpu;
 import android.graphics.SurfaceTexture;
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
 import android.util.Size;
 import android.view.SurfaceHolder;
 import android.view.SurfaceView;
 import android.view.View;
 import android.view.ViewGroup;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
 import com.google.mediapipe.components.CameraHelper;
 import com.google.mediapipe.components.CameraXPreviewHelper;
 import com.google.mediapipe.components.ExternalTextureConverter;
 import com.google.mediapipe.components.FrameProcessor;
 import com.google.mediapipe.components.PermissionHelper;
 import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.glutil.EglManager;
+import java.util.List;
 
 /** Main activity of MediaPipe example apps. */
 public class MainActivity extends AppCompatActivity {
@@ -37,6 +41,7 @@ public class MainActivity extends AppCompatActivity {
   private static final String BINARY_GRAPH_NAME = "objectdetectiongpu.binarypb";
   private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
   private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final String OUTPUT_DETECTIONS_STREAM_NAME = "output_detections";
   private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
 
   // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
@@ -90,6 +95,14 @@ public class MainActivity extends AppCompatActivity {
             OUTPUT_VIDEO_STREAM_NAME);
     processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
 
+    processor.addPacketCallback(
+        OUTPUT_DETECTIONS_STREAM_NAME,
+        (packet) -> {
+          Log.d(TAG, "Received detections packet.");
+          List<Detection> detections = PacketGetter.getProtoVector(packet, Detection.parser());
+          Log.d(TAG, "[TS:" + packet.getTimestamp() + "] " + getDetectionsDebugString(detections));
+        });
+
     PermissionHelper.checkAndRequestCameraPermissions(this);
   }
 
@@ -138,12 +151,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override
@@ -164,4 +180,22 @@ public class MainActivity extends AppCompatActivity {
         });
     cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
   }
+
+  private static String getDetectionsDebugString(List<Detection> detections) {
+    if (detections.isEmpty()) {
+      return "No detections";
+    }
+    String detectionsStr = "Number of objects detected: " + detections.size() + "\n";
+    int objectIndex = 0;
+    for (Detection detection : detections) {
+      detectionsStr += "\t#Object[" + objectIndex + "]: \n";
+      List<String> labels = detection.getLabelList();
+      List<Float> scores = detection.getScoreList();
+      for (int i = 0; i < labels.size(); ++i) {
+        detectionsStr += "\t\tLabel [" + i + "]: " + labels.get(i) + ", " + scores.get(i) + "\n";
+      }
+      ++objectIndex;
+    }
+    return detectionsStr;
+  }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/MainActivity.java

@@ -138,12 +138,15 @@ public class MainActivity extends AppCompatActivity {
                 // based on the size of the SurfaceView that contains the display.
                 Size viewSize = new Size(width, height);
                 Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+                boolean isCameraRotated = cameraHelper.isCameraRotated();
 
                 // Connect the converter to the camera-preview frames as its input (via
                 // previewFrameTexture), and configure the output width and height as the computed
                 // display size.
                 converter.setSurfaceTextureAndAttachToGLContext(
-                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+                    previewFrameTexture,
+                    isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
+                    isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
               }
 
               @Override