Project import generated by Copybara.

GitOrigin-RevId: 2146b10f0a498f665f246e16033b686c7947b92d

Dockerfile

@@ -23,6 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
+        gcc-8 g++-8 \
         ca-certificates \
         curl \
         ffmpeg \
@@ -44,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 --slave /usr/bin/g++ g++ /usr/bin/g++-8
 RUN pip3 install --upgrade setuptools
 RUN pip3 install wheel
 RUN pip3 install future
@@ -54,7 +56,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.4.1
+ARG BAZEL_VERSION=3.7.2
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

MANIFEST.in

@@ -10,3 +10,10 @@ include requirements.txt
 recursive-include mediapipe/modules *.tflite *.txt *.binarypb
 exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
 exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite
+exclude mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite
+exclude mediapipe/modules/pose_landmark/pose_landmark_lite.tflite
+exclude mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite

README.md

@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

WORKSPACE

@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
+    urls = [
-    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+    ],
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
 )
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "3.4.0")
+versions.check(minimum_bazel_version = "3.7.2")
-
 
 # ABSL cpp library lts_2020_09_23
 http_archive(
@@ -38,8 +41,8 @@ http_archive(
 
 http_archive(
    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-main",
+   strip_prefix = "rules_foreign_cc-0.1.0",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
+   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -117,7 +120,8 @@ http_archive(
 # libyuv
 http_archive(
     name = "libyuv",
-    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
+    # Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8.
+    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"],
     build_file = "@//third_party:libyuv.BUILD",
 )
 
@@ -304,8 +308,8 @@ http_archive(
 
 # Maven dependencies.
 
-RULES_JVM_EXTERNAL_TAG = "3.2"
+RULES_JVM_EXTERNAL_TAG = "4.0"
-RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
 
 http_archive(
     name = "rules_jvm_external",
@@ -318,7 +322,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 
 # Important: there can only be one maven_install rule. Add new maven deps here.
 maven_install(
-    name = "maven",
     artifacts = [
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.2.0",
@@ -334,6 +337,8 @@ maven_install(
         "androidx.test.espresso:espresso-core:3.1.1",
         "com.github.bumptech.glide:glide:4.11.0",
         "com.google.android.material:material:aar:1.0.0-rc01",
+        "com.google.auto.value:auto-value:1.6.4",
+        "com.google.auto.value:auto-value-annotations:1.6.4",
         "com.google.code.findbugs:jsr305:3.0.2",
         "com.google.flogger:flogger-system-backend:0.3.1",
         "com.google.flogger:flogger:0.3.1",
@@ -343,10 +348,10 @@ maven_install(
         "org.hamcrest:hamcrest-library:1.3",
     ],
     repositories = [
-        "https://jcenter.bintray.com",
         "https://maven.google.com",
         "https://dl.google.com/dl/android/maven2",
         "https://repo1.maven.org/maven2",
+        "https://jcenter.bintray.com",
     ],
     fetch_sources = True,
     version_conflict_policy = "pinned",
@@ -363,10 +368,10 @@ http_archive(
     ],
 )
 
-#Tensorflow repo should always go after the other external dependencies.
+# Tensorflow repo should always go after the other external dependencies.
-# 2020-12-09
+# 2021-04-30
-_TENSORFLOW_GIT_COMMIT = "0eadbb13cef1226b1bae17c941f7870734d97f8a"
+_TENSORFLOW_GIT_COMMIT = "5bd3c57ef184543d22e34e36cff9d9bea608e06d"
-_TENSORFLOW_SHA256= "4ae06daa5b09c62f31b7bc1f781fd59053f286dd64355830d8c2ac601b795ef0"
+_TENSORFLOW_SHA256= "9a45862834221aafacf6fb275f92b3876bc89443cbecc51be93f13839a6609f0"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -383,5 +388,7 @@ http_archive(
     sha256 = _TENSORFLOW_SHA256,
 )
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
-tf_workspace(tf_repo_name = "org_tensorflow")
+tf_workspace3()
+load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+tf_workspace2()

build_desktop_examples.sh

@@ -17,15 +17,15 @@
 # Script to build/run all MediaPipe desktop example apps (with webcam input).
 #
 # To build and run all apps and store them in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir
+#   $ ./build_desktop_examples.sh -d out_dir
 #   Omitting -d and the associated directory saves all generated apps in the
 #   current directory.
 # To build all apps and store them in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir -b
+#   $ ./build_desktop_examples.sh -d out_dir -b
 #   Omitting -d and the associated directory saves all generated apps in the
 #   current directory.
 # To run all apps already stored in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir -r
+#   $ ./build_desktop_examples.sh -d out_dir -r
 #   Omitting -d and the associated directory assumes all apps are in the current
 #   directory.
 

docs/framework_concepts/calculators.md

@@ -187,7 +187,7 @@ node {
 ```
 
 In the calculator implementation, inputs and outputs are also identified by tag
-name and index number. In the function below input are output are identified:
+name and index number. In the function below input and output are identified:
 
 *   By index number: The combined input stream is identified simply by index
     `0`.
@@ -355,7 +355,6 @@ class PacketClonerCalculator : public CalculatorBase {
               current_[i].At(cc->InputTimestamp()));
           // Add a packet to output stream of index i a packet from inputstream i
           // with timestamp common to all present inputs
-          //
         } else {
           cc->Outputs().Index(i).SetNextTimestampBound(
               cc->InputTimestamp().NextAllowedInStream());
@@ -382,7 +381,7 @@ defined your calculator class, register it with a macro invocation
 REGISTER_CALCULATOR(calculator_class_name).
 
 Below is a trivial MediaPipe graph that has 3 input streams, 1 node
-(PacketClonerCalculator) and 3 output streams.
+(PacketClonerCalculator) and 2 output streams.
 
 ```proto
 input_stream: "room_mic_signal"

docs/framework_concepts/framework_concepts.md

@@ -110,3 +110,12 @@ Other policies are also available, implemented using a separate kind of
 component known as an InputStreamHandler.
 
 See [Synchronization](synchronization.md) for more details.
+
+### Realtime data streams
+
+MediaPipe calculator graphs are often used to process streams of video or audio
+frames for interactive applications. Normally, each Calculator runs as soon as
+all of its input packets for a given timestamp become available. Calculators
+used in realtime graphs need to define output timestamp bounds based on input
+timestamp bounds in order to allow downstream calculators to be scheduled
+promptly. See [Realtime data streams](realtime.md) for details.

docs/framework_concepts/graphs.md

@@ -83,12 +83,12 @@ Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
     output_stream: "out3"
 
     node {
-        calculator: "PassThroughculator"
+        calculator: "PassThroughCalculator"
         input_stream: "out1"
         output_stream: "out2"
     }
     node {
-        calculator: "PassThroughculator"
+        calculator: "PassThroughCalculator"
         input_stream: "out2"
         output_stream: "out3"
     }

docs/framework_concepts/packets.md

@@ -12,19 +12,30 @@ nav_order: 3
 {:toc}
 ---
 
-Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
+Calculators communicate by sending and receiving packets. Typically a single
+packet is sent along each input stream at each input timestamp. A packet can
+contain any kind of data, such as a single frame of video or a single integer
+detection count.
 
 ## Creating a packet
 
-Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
+Packets are generally created with `mediapipe::MakePacket<T>()` or
+`mediapipe::Adopt()` (from packet.h).
 
 ```c++
-// Create some data.
+// Create a packet containing some new data.
-auto data = absl::make_unique<MyDataClass>("constructor_argument");
+Packet p = MakePacket<MyDataClass>("constructor_argument");
-// Create a packet to own the data.
-Packet p = Adopt(data.release());
 // Make a new packet with the same data and a different timestamp.
 Packet p2 = p.At(Timestamp::PostStream());
 ```
 
+or:
+
+```c++
+// Create some new data.
+auto data = absl::make_unique<MyDataClass>("constructor_argument");
+// Create a packet to own the data.
+Packet p = Adopt(data.release()).At(Timestamp::PostStream());
+```
+
 Data within a packet is accessed with `Packet::Get<T>()`

docs/framework_concepts/realtime.md

@@ -0,0 +1,187 @@
+---
+layout: default
+title: Processing real-time data streams
+nav_order: 6
+has_children: true
+has_toc: false
+---
+
+# Processing real-time data streams
+{: .no_toc }
+
+1. TOC
+{:toc}
+---
+
+## Realtime timestamps
+
+MediaPipe calculator graphs are often used to process streams of video or audio
+frames for interactive applications. The MediaPipe framework requires only that
+successive packets be assigned monotonically increasing timestamps. By
+convention, realtime calculators and graphs use the recording time or the
+presentation time of each frame as its timestamp, with each timestamp indicating
+the microseconds since `Jan/1/1970:00:00:00`. This allows packets from various
+sources to be processed in a globally consistent sequence.
+
+## Realtime scheduling
+
+Normally, each Calculator runs as soon as all of its input packets for a given
+timestamp become available. Normally, this happens when the calculator has
+finished processing the previous frame, and each of the calculators producing
+its inputs have finished processing the current frame. The MediaPipe scheduler
+invokes each calculator as soon as these conditions are met. See
+[Synchronization](synchronization.md) for more details.
+
+## Timestamp bounds
+
+When a calculator does not produce any output packets for a given timestamp, it
+can instead output a "timestamp bound" indicating that no packet will be
+produced for that timestamp. This indication is necessary to allow downstream
+calculators to run at that timestamp, even though no packet has arrived for
+certain streams for that timestamp. This is especially important for realtime
+graphs in interactive applications, where it is crucial that each calculator
+begin processing as soon as possible.
+
+Consider a graph like the following:
+
+```
+node {
+   calculator: "A"
+   input_stream: "alpha_in"
+   output_stream: "alpha"
+}
+node {
+   calculator: "B"
+   input_stream: "alpha"
+   input_stream: "foo"
+   output_stream: "beta"
+}
+```
+
+Suppose: at timestamp `T`, node `A` doesn't send a packet in its output stream
+`alpha`. Node `B` gets a packet in `foo` at timestamp `T` and is waiting for a
+packet in `alpha` at timestamp `T`. If `A` doesn't send `B` a timestamp bound
+update for `alpha`, `B` will keep waiting for a packet to arrive in `alpha`.
+Meanwhile, the packet queue of `foo` will accumulate packets at `T`, `T+1` and
+so on.
+
+To output a packet on a stream, a calculator uses the API functions
+`CalculatorContext::Outputs` and `OutputStream::Add`. To instead output a
+timestamp bound on a stream, a calculator can use the API functions
+`CalculatorContext::Outputs` and `CalculatorContext::SetNextTimestampBound`. The
+specified bound is the lowest allowable timestamp for the next packet on the
+specified output stream. When no packet is output, a calculator will typically
+do something like:
+
+```
+cc->Outputs().Tag("output_frame").SetNextTimestampBound(
+  cc->InputTimestamp().NextAllowedInStream());
+```
+
+The function `Timestamp::NextAllowedInStream` returns the successive timestamp.
+For example, `Timestamp(1).NextAllowedInStream() == Timestamp(2)`.
+
+## Propagating timestamp bounds
+
+Calculators that will be used in realtime graphs need to define output timestamp
+bounds based on input timestamp bounds in order to allow downstream calculators
+to be scheduled promptly. A common pattern is for calculators to output packets
+with the same timestamps as their input packets. In this case, simply outputting
+a packet on every call to `Calculator::Process` is sufficient to define output
+timestamp bounds.
+
+However, calculators are not required to follow this common pattern for output
+timestamps, they are only required to choose monotonically increasing output
+timestamps. As a result, certain calculators must calculate timestamp bounds
+explicitly. MediaPipe provides several tools for computing appropriate timestamp
+bound for each calculator.
+
+1\. **SetNextTimestampBound()** can be used to specify the timestamp bound, `t +
+1`, for an output stream.
+
+```
+cc->Outputs.Tag("OUT").SetNextTimestampBound(t.NextAllowedInStream());
+```
+
+Alternatively, an empty packet with timestamp `t` can be produced to specify the
+timestamp bound `t + 1`.
+
+```
+cc->Outputs.Tag("OUT").Add(Packet(), t);
+```
+
+The timestamp bound of an input stream is indicated by the packet or the empty
+packet on the input stream.
+
+```
+Timestamp bound = cc->Inputs().Tag("IN").Value().Timestamp();
+```
+
+2\. **TimestampOffset()** can be specified in order to automatically copy the
+timestamp bound from input streams to output streams.
+
+```
+cc->SetTimestampOffset(0);
+```
+
+This setting has the advantage of propagating timestamp bounds automatically,
+even when only timestamp bounds arrive and Calculator::Process is not invoked.
+
+3\. **ProcessTimestampBounds()** can be specified in order to invoke
+`Calculator::Process` for each new "settled timestamp", where the "settled
+timestamp" is the new highest timestamp below the current timestamp bounds.
+Without `ProcessTimestampBounds()`, `Calculator::Process` is invoked only with
+one or more arriving packets.
+
+```
+cc->SetProcessTimestampBounds(true);
+```
+
+This setting allows a calculator to perform its own timestamp bounds calculation
+and propagation, even when only input timestamps are updated. It can be used to
+replicate the effect of `TimestampOffset()`, but it can also be used to
+calculate a timestamp bound that takes into account additional factors.
+
+For example, in order to replicate `SetTimestampOffset(0)`, a calculator could
+do the following:
+
+```
+absl::Status Open(CalculatorContext* cc) {
+  cc->SetProcessTimestampBounds(true);
+}
+
+absl::Status Process(CalculatorContext* cc) {
+  cc->Outputs.Tag("OUT").SetNextTimestampBound(
+      cc->InputTimestamp().NextAllowedInStream());
+}
+```
+
+## Scheduling of Calculator::Open and Calculator::Close
+
+`Calculator::Open` is invoked when all required input side-packets have been
+produced. Input side-packets can be provided by the enclosing application or by
+"side-packet calculators" inside the graph. Side-packets can be specified from
+outside the graph using the API's `CalculatorGraph::Initialize` and
+`CalculatorGraph::StartRun`. Side packets can be specified by calculators within
+the graph using `CalculatorGraphConfig::OutputSidePackets` and
+`OutputSidePacket::Set`.
+
+Calculator::Close is invoked when all of the input streams have become `Done` by
+being closed or reaching timestamp bound `Timestamp::Done`.
+
+**Note:** If the graph finishes all pending calculator execution and becomes
+`Done`, before some streams become `Done`, then MediaPipe will invoke the
+remaining calls to `Calculator::Close`, so that every calculator can produce its
+final outputs.
+
+The use of `TimestampOffset` has some implications for `Calculator::Close`. A
+calculator specifying `SetTimestampOffset(0)` will by design signal that all of
+its output streams have reached `Timestamp::Done` when all of its input streams
+have reached `Timestamp::Done`, and therefore no further outputs are possible.
+This prevents such a calculator from emitting any packets during
+`Calculator::Close`. If a calculator needs to produce a summary packet during
+`Calculator::Close`, `Calculator::Process` must specify timestamp bounds such
+that at least one timestamp (such as `Timestamp::Max`) remains available during
+`Calculator::Close`. This means that such a calculator normally cannot rely upon
+`SetTimestampOffset(0)` and must instead specify timestamp bounds explicitly
+using `SetNextTimestampBounds()`.

docs/getting_started/android.md

@@ -28,7 +28,7 @@ Gradle.
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
 *   Setup Android SDK release 28.0.3 and above.
-*   Setup Android NDK r18b and above.
+*   Setup Android NDK version between 18 and 21.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
 below for Android Studio setup). However, if you prefer using MediaPipe without
@@ -57,7 +57,7 @@ Please verify all the necessary packages are installed.
 *   Android SDK Build-Tools 28 or 29
 *   Android SDK Platform-Tools 28 or 29
 *   Android SDK Tools 26.1.1
-*   Android NDK 17c or above
+*   Android NDK 19c or above
 
 ### Option 1: Build with Bazel in Command Line
 
@@ -111,7 +111,7 @@ app:
     *   Verify that Android SDK Build-Tools 28 or 29 is installed.
     *   Verify that Android SDK Platform-Tools 28 or 29 is installed.
     *   Verify that Android SDK Tools 26.1.1 is installed.
-    *   Verify that Android NDK 17c or above is installed.
+    *   Verify that Android NDK 19c or above is installed.
     *   Take note of the Android NDK Location, e.g.,
         `/usr/local/home/Android/Sdk/ndk-bundle` or
         `/usr/local/home/Android/Sdk/ndk/20.0.5594570`.

docs/getting_started/android_archive_library.md

@@ -37,7 +37,7 @@ each project.
     load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar")
 
     mediapipe_aar(
-        name = "mp_face_detection_aar",
+        name = "mediapipe_face_detection",
         calculators = ["//mediapipe/graphs/face_detection:mobile_calculators"],
     )
     ```
@@ -45,26 +45,29 @@ each project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+    bazel build -c opt --strip=ALWAYS \
-        --fat_apk_cpu=arm64-v8a,armeabi-v7a --strip=ALWAYS \
+        --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
-        //path/to/the/aar/build/file:aar_name
+        --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //path/to/the/aar/build/file:aar_name.aar
     ```
 
-    For the face detection AAR target we made in the step 1, run:
+    For the face detection AAR target we made in step 1, run:
 
     ```bash
-    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+    bazel build -c opt --strip=ALWAYS \
-        //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
+        --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+        --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar
 
     # It should print:
-    # Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar up-to-date:
+    # Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar up-to-date:
-    # bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    # bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     ```
 
 3.  (Optional) Save the AAR to your preferred location.
 
     ```bash
-    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     /absolute/path/to/your/preferred/location
     ```
 
@@ -75,7 +78,7 @@ each project.
 2.  Copy the AAR into app/libs.
 
     ```bash
-    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     /path/to/your/app/libs/
     ```
 
@@ -92,29 +95,14 @@ each project.
     [the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite).
 
     ```bash
-    bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
+    bazel build -c opt mediapipe/graphs/face_detection:face_detection_mobile_gpu_binary_graph
-    cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
+    cp bazel-bin/mediapipe/graphs/face_detection/face_detection_mobile_gpu.binarypb /path/to/your/app/src/main/assets/
     cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/
     ```
 
     ![Screenshot](../images/mobile/assets_location.png)
 
-4.  Make app/src/main/jniLibs and copy OpenCV JNI libraries into
+4.  Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
-    app/src/main/jniLibs.
-
-    MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
-    files into app/src/main/jniLibs. You can download the official OpenCV
-    Android SDK from
-    [here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
-    and run:
-
-    ```bash
-    cp -R ~/Downloads/OpenCV-android-sdk/sdk/native/libs/arm* /path/to/your/app/src/main/jniLibs/
-    ```
-
-    ![Screenshot](../images/mobile/android_studio_opencv_location.png)
-
-5.  Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
 
     ```
     dependencies {
@@ -136,10 +124,14 @@ each project.
         implementation "androidx.camera:camera-core:$camerax_version"
         implementation "androidx.camera:camera-camera2:$camerax_version"
         implementation "androidx.camera:camera-lifecycle:$camerax_version"
+        // AutoValue
+        def auto_value_version = "1.6.4"
+        implementation "com.google.auto.value:auto-value-annotations:$auto_value_version"
+        annotationProcessor "com.google.auto.value:auto-value:$auto_value_version"
     }
     ```
 
-6.  Follow our Android app examples to use MediaPipe in Android Studio for your
+5.  Follow our Android app examples to use MediaPipe in Android Studio for your
     use case. If you are looking for an example, a face detection example can be
     found
     [here](https://github.com/jiuqiant/mediapipe_face_detection_aar_example) and

docs/getting_started/install.md

@@ -25,25 +25,11 @@ install --user six`.
 
 ## Installing on Debian and Ubuntu
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
-
-    For Nvidia Jetson and Raspberry Pi devices with aarch64 Linux, Bazel needs
-    to be built from source:
-
-    ```bash
-    # For Bazel 3.4.1
-    mkdir $HOME/bazel-3.4.1
-    cd $HOME/bazel-3.4.1
-    wget https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-dist.zip
-    sudo apt-get install build-essential openjdk-8-jdk python zip unzip
-    unzip bazel-3.4.1-dist.zip
-    env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
-    sudo cp output/bazel /usr/local/bin/
-    ```
 
 2.  Checkout MediaPipe repository.
 
@@ -207,11 +193,11 @@ build issues.
 
 **Disclaimer**: Running MediaPipe on CentOS is experimental.
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -336,11 +322,11 @@ build issues.
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
         Tools by `xcode-select --install`.
 
-2.  Install Bazel.
+2.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
 
 3.  Checkout MediaPipe repository.
 
@@ -353,7 +339,7 @@ build issues.
 4.  Install OpenCV and FFmpeg.
 
     Option 1. Use HomeBrew package manager tool to install the pre-compiled
-    OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
+    OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
 
     ```bash
     $ brew install opencv@3
@@ -484,29 +470,36 @@ next section.
 
 4.  Install Visual C++ Build Tools 2019 and WinSDK
 
-    Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
+    Go to
-    build tools, and install Microsoft Visual C++ 2019 Redistributable and
+    [the VisualStudio website](https://visualstudio.microsoft.com/visual-cpp-build-tools),
-    Microsoft Build Tools 2019.
+    download build tools, and install Microsoft Visual C++ 2019 Redistributable
+    and Microsoft Build Tools 2019.
 
     Download the WinSDK from
-    https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
+    [the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
-    install.
+    and install.
 
-5.  Install Bazel and add the location of the Bazel executable to the `%PATH%`
+5.  Install Bazel or Bazelisk and add the location of the Bazel executable to
-    environment variable.
+    the `%PATH%` environment variable.
 
-    Follow the official
+    Option 1. Follow
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
+    [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.4 or higher.
+    to install Bazel 3.7.2 or higher.
 
-6.  Set Bazel variables.
+    Option 2. Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
+
+6.  Set Bazel variables. Learn more details about
+    ["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
+    in the Bazel official documentation.
 
     ```
-    # Find the exact paths and version numbers from your local version.
+    # Please find the exact paths and version numbers from your local version.
     C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
     C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
-    C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
+    C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
-    C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
+    C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
     ```
 
 7.  Checkout MediaPipe repository.
@@ -593,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel.
+5.  Install Bazelisk.
 
-    ```bash
+    Follow the official
-    username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    https://storage.googleapis.com/bazel/3.4.1/release/bazel-3.4.1-installer-linux-x86_64.sh && \
+    to install Bazelisk.
-    sudo mkdir -p /usr/local/bazel/3.4.1 && \
-    chmod 755 bazel-3.4.1-installer-linux-x86_64.sh && \
-    sudo ./bazel-3.4.1-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.1 && \
-    source /usr/local/bazel/3.4.1/lib/bazel/bin/bazel-complete.bash
-
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.1/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/3.4.1/lib/bazel/bin/bazel'
-    ```
 
 6.  Checkout MediaPipe repository.
 
@@ -753,7 +738,7 @@ common build issues.
     root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk.sh
 
     # Should print:
-    # Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b
+    # Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r19c
     # Set android_ndk_repository and android_sdk_repository in WORKSPACE
     # Done
 

docs/getting_started/javascript.md

@@ -72,9 +72,9 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 [Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
 [H-npm]: https://www.npmjs.com/package/@mediapipe/hands
 [P-npm]: https://www.npmjs.com/package/@mediapipe/pose
-[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
-[cam-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
-[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
 [Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
 [F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
 [Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection

docs/getting_started/python.md

@@ -26,7 +26,7 @@ You can, for instance, activate a Python virtual environment:
 $ python3 -m venv mp_env && source mp_env/bin/activate
 ```
 
-Install MediaPipe Python package and start Python intepreter:
+Install MediaPipe Python package and start Python interpreter:
 
 ```bash
 (mp_env)$ pip install mediapipe

docs/getting_started/troubleshooting.md

@@ -97,6 +97,49 @@ linux_opencv/macos_opencv/windows_opencv.BUILD files for your local opencv
 libraries. [This GitHub issue](https://github.com/google/mediapipe/issues/666)
 may also help.
 
+## Python pip install failure
+
+The error message:
+
+```
+ERROR: Could not find a version that satisfies the requirement mediapipe
+ERROR: No matching distribution found for mediapipe
+```
+
+after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system.
+Please note that MediaPipe Python PyPI officially supports the **64-bit**
+version of Python 3.7 and above on the following OS:
+
+-   x86_64 Linux
+-   x86_64 macOS 10.15+
+-   amd64 Windows
+
+If the OS is currently supported and you still see this error, please make sure
+that both the Python and pip binary are for Python 3.7 and above. Otherwise,
+please consider building the MediaPipe Python package locally by following the
+instructions [here](python.md#building-mediapipe-python-package).
+
+## Python DLL load failure on Windows
+
+The error message:
+
+```
+ImportError: DLL load failed: The specified module could not be found
+```
+
+usually indicates that the local Windows system is missing Visual C++
+redistributable packages and/or Visual C++ runtime DLLs. This can be solved by
+either installing the official
+[vc_redist.x64.exe](https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0)
+or installing the "msvc-runtime" Python package by running
+
+```bash
+$ python -m pip install msvc-runtime
+```
+
+Please note that the "msvc-runtime" Python package is not released or maintained
+by Microsoft.
+
 ## Native method not found
 
 The error message:

docs/index.md

@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/solutions/face_detection.md

@@ -77,7 +77,8 @@ Supported configuration options:
 ```python
 import cv2
 import mediapipe as mp
-mp_face_detction = mp.solutions.face_detection
+mp_face_detection = mp.solutions.face_detection
+mp_drawing = mp.solutions.drawing_utils
 
 # For static images:
 with mp_face_detection.FaceDetection(
@@ -183,8 +184,8 @@ function onResults(results) {
   canvasCtx.restore();
 }
 
-const faceDetection = new Objectron({locateFile: (file) => {
+const faceDetection = new FaceDetection({locateFile: (file) => {
-  return `https://cdn.jsdelivr.net/npm/@mediapipe/objectr[email protected]/${file}`;
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detecti[email protected]/${file}`;
 }});
 faceDetection.setOptions({
   minDetectionConfidence: 0.5

docs/solutions/holistic.md

@@ -135,12 +135,11 @@ another detection until it loses track, on reducing computation and latency. If
 set to `true`, person detection runs every input image, ideal for processing a
 batch of static, possibly unrelated, images. Default to `false`.
 
-#### upper_body_only
+#### model_complexity
 
-If set to `true`, the solution outputs only the 25 upper-body pose landmarks
+Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
-(535 in total) instead of the full set of 33 pose landmarks (543 in total). Note
+well as inference latency generally go up with the model complexity. Default to
-that upper-body-only prediction may be more accurate for use cases where the
+`1`.
-lower-body parts are mostly out of view. Default to `false`.
 
 #### smooth_landmarks
 
@@ -207,7 +206,7 @@ install MediaPipe Python package, then learn more in the companion
 Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
-*   [upper_body_only](#upper_body_only)
+*   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
@@ -219,7 +218,9 @@ mp_drawing = mp.solutions.drawing_utils
 mp_holistic = mp.solutions.holistic
 
 # For static images:
-with mp_holistic.Holistic(static_image_mode=True) as holistic:
+with mp_holistic.Holistic(
+    static_image_mode=True,
+    model_complexity=2) as holistic:
   for idx, file in enumerate(file_list):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -240,8 +241,6 @@ with mp_holistic.Holistic(static_image_mode=True) as holistic:
         annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
     mp_drawing.draw_landmarks(
         annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-    # Use mp_holistic.UPPER_BODY_POSE_CONNECTIONS for drawing below when
-    # upper_body_only is set to True.
     mp_drawing.draw_landmarks(
         annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
@@ -291,7 +290,7 @@ and the following usage example.
 
 Supported configuration options:
 
-*   [upperBodyOnly](#upper_body_only)
+*   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
@@ -348,7 +347,7 @@ const holistic = new Holistic({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`;
 }});
 holistic.setOptions({
-  upperBodyOnly: false,
+  modelComplexity: 1,
   smoothLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5

docs/solutions/models.md

@@ -15,10 +15,10 @@ nav_order: 30
 ### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
 
 *   Face detection model for front-facing/selfie camera:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite),
     [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
 *   Face detection model for back-facing camera:
-    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
+    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back.tflite)
 *   [Model card](https://mediapipe.page.link/blazeface-mc)
 
 ### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
@@ -49,10 +49,10 @@ nav_order: 30
 
 *   Pose detection model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
-*   Full-body pose landmark model:
+*   Pose landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite)
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite),
-*   Upper-body pose landmark model:
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full.tflite),
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
+    [TFLite model (heavy)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite)
 *   [Model card](https://mediapipe.page.link/blazepose-mc)
 
 ### [Holistic](https://google.github.io/mediapipe/solutions/holistic)

docs/solutions/objectron.md

@@ -358,15 +358,17 @@ cap.release()
 ## Example Apps
 
 Please first see general instructions for
-[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
+[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
-how to build MediaPipe examples.
+[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
 
 Note: To visualize a graph, copy the graph and paste it into
 [MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Two-stage Objectron
+### Mobile
+
+#### Two-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -404,7 +406,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Single-stage Objectron
+#### Single-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -428,7 +430,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Assets
+#### Assets
 
 Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
  format into a custom .uuu format. This can be done for user assets as follows:
@@ -449,9 +451,35 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
 > absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
 
-### Coordinate Systems
 
-#### Object Coordinate
+### Desktop
+
+To build the application, run:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
+```
+
+To run the application, replace `<input video path>` and `<output video path>`
+in the command below with your own paths, and `<landmark model path>` and
+`<allowed labels>` with the following:
+
+Category | `<landmark model path>`                                                     | `<allowed labels>`
+:------- | :-------------------------------------------------------------------------- | :-----------------
+Shoe     | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
+Chair    | mediapipe/modules/objectron/object_detection_3d_chair.tflite    | Chair
+Cup      | mediapipe/modules/objectron/object_detection_3d_cup.tflite      | Mug
+Camera   | mediapipe/modules/objectron/object_detection_3d_camera.tflite   | Camera
+
+```
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
+  --input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
+```
+
+## Coordinate Systems
+
+### Object Coordinate
 
 Each object has its object coordinate frame. We use the below object coordinate
 definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
@@ -459,7 +487,7 @@ origin is at the center of the 3D bounding box.
 
 ![box_coordinate.svg](../images/box_coordinate.svg)
 
-#### Camera Coordinate
+### Camera Coordinate
 
 A 3D object is parameterized by its `scale` and `rotation`, `translation` with
 regard to the camera coordinate frame. In this API we use the below camera
@@ -476,7 +504,7 @@ camera frame by applying `rotation` and `translation`:
 landmarks_3d = rotation * scale * unit_box + translation
 ```
 
-#### NDC Space
+### NDC Space
 
 In this API we use
 [NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
@@ -495,7 +523,7 @@ y_ndc = -fy * Y / Z + py
 z_ndc = 1 / Z
 ```
 
-#### Pixel Space
+### Pixel Space
 
 In this API we set upper-left coner of an image as the origin of pixel
 coordinate. One can convert from NDC to pixel space as follows:
@@ -532,10 +560,11 @@ py = -py_pixel * 2.0 / image_height + 1.0
     [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
 *   Google AI Blog:
     [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
+*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
 *   Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
     Shape Supervision](https://arxiv.org/abs/2003.03522)
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
-    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
+    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
 *   [Models and model cards](./models.md#objectron)
 *   [Python Colab](https://mediapipe.page.link/objectron_py_colab)

docs/solutions/pose.md

@@ -2,6 +2,8 @@
 layout: default
 title: Pose
 parent: Solutions
+has_children: true
+has_toc: false
 nav_order: 5
 ---
 
@@ -21,16 +23,14 @@ nav_order: 5
 ## Overview
 
 Human pose estimation from video plays a critical role in various applications
-such as
+such as [quantifying physical exercises](./pose_classification.md), sign
-[quantifying physical exercises](#pose-classification-and-repetition-counting),
+language recognition, and full-body gesture control. For example, it can form
-sign language recognition, and full-body gesture control. For example, it can
+the basis for yoga, dance, and fitness applications. It can also enable the
-form the basis for yoga, dance, and fitness applications. It can also enable the
 overlay of digital content and information on top of the physical world in
 augmented reality.
 
 MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
-33 3D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
+33 3D landmarks on the whole body from RGB video frames utilizing our
-frames utilizing our
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 research that also powers the
 [ML Kit Pose Detection API](https://developers.google.com/ml-kit/vision/pose-detection).
@@ -39,9 +39,9 @@ environments for inference, whereas our method achieves real-time performance on
 most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
 [python](#python-solution-api) and even on the [web](#javascript-solution-api).
 
-![pose_tracking_upper_body_example.gif](../images/mobile/pose_tracking_upper_body_example.gif) |
+![pose_tracking_example.gif](../images/mobile/pose_tracking_example.gif) |
-:--------------------------------------------------------------------------------------------: |
+:----------------------------------------------------------------------: |
-*Fig 1. Example of MediaPipe Pose for upper-body pose tracking.*                               |
+*Fig 1. Example of MediaPipe Pose for pose tracking.*                    |
 
 ## ML Pipeline
 
@@ -76,6 +76,36 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
+## Pose Estimation Quality
+
+To evaluate the quality of our [models](./models.md#pose) against other
+well-performing publicly available solutions, we use three different validation
+datasets, representing different verticals: Yoga, Dance and HIIT. Each image
+contains only a single person located 2-4 meters from the camera. To be
+consistent with other solutions, we perform evaluation only for 17 keypoints
+from [COCO topology](https://cocodataset.org/#keypoints-2020).
+
+Method                                                                                                | Yoga <br/> [`mAP`] | Yoga <br/> [`[email protected]`] | Dance <br/> [`mAP`] | Dance <br/> [`[email protected]`] | HIIT <br/> [`mAP`] | HIIT <br/> [`[email protected]`]
+----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------:
+BlazePose.Heavy                                                                                       | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
+BlazePose.Full                                                                                        | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
+BlazePose.Lite                                                                                        | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
+[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
+[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
+
+![pose_tracking_pck_chart.png](../images/mobile/pose_tracking_pck_chart.png) |
+:--------------------------------------------------------------------------: |
+*Fig 2. Quality evaluation in [`[email protected]`].*                                  |
+
+We designed our models specifically for live perception use cases, so all of
+them work in real-time on the majority of modern devices.
+
+Method          | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
+--------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
+BlazePose.Heavy | 53 ms                                                                                        | 38 ms
+BlazePose.Full  | 25 ms                                                                                        | 27 ms
+BlazePose.Lite  | 20 ms                                                                                        | 25 ms
+
 ## Models
 
 ### Person/pose Detection Model (BlazePose Detector)
@@ -92,15 +122,12 @@ hip midpoints.
 
 ![pose_tracking_detector_vitruvian_man.png](../images/mobile/pose_tracking_detector_vitruvian_man.png) |
 :----------------------------------------------------------------------------------------------------: |
-*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
+*Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
 
 ### Pose Landmark Model (BlazePose GHUM 3D)
 
-The landmark model in MediaPipe Pose comes in two versions: a full-body model
+The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
-that predicts the location of 33 pose landmarks (see figure below), and an
+(see figure below).
-upper-body version that only predicts the first 25. The latter may be more
-accurate than the former in scenarios where the lower-body parts are mostly out
-of view.
 
 Please find more detail in the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
@@ -110,7 +137,7 @@ this [paper](https://arxiv.org/abs/2006.10204) and
 
 ![pose_tracking_full_body_landmarks.png](../images/mobile/pose_tracking_full_body_landmarks.png) |
 :----------------------------------------------------------------------------------------------: |
-*Fig 3. 33 pose landmarks.*                                                                      |
+*Fig 4. 33 pose landmarks.*                                                                      |
 
 ## Solution APIs
 
@@ -128,12 +155,11 @@ until it loses track, on reducing computation and latency. If set to `true`,
 person detection runs every input image, ideal for processing a batch of static,
 possibly unrelated, images. Default to `false`.
 
-#### upper_body_only
+#### model_complexity
 
-If set to `true`, the solution outputs only the 25 upper-body pose landmarks.
+Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
-Otherwise, it outputs the full set of 33 pose landmarks. Note that
+well as inference latency generally go up with the model complexity. Default to
-upper-body-only prediction may be more accurate for use cases where the
+`1`.
-lower-body parts are mostly out of view. Default to `false`.
 
 #### smooth_landmarks
 
@@ -169,9 +195,6 @@ A list of pose landmarks. Each lanmark consists of the following:
     being the origin, and the smaller the value the closer the landmark is to
     the camera. The magnitude of `z` uses roughly the same scale as `x`.
 
-    Note: `z` is predicted only in full-body mode, and should be discarded when
-    [upper_body_only](#upper_body_only) is `true`.
-
 *   `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the
     landmark being visible (present and not occluded) in the image.
 
@@ -184,7 +207,7 @@ install MediaPipe Python package, then learn more in the companion
 Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
-*   [upper_body_only](#upper_body_only)
+*   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
@@ -197,7 +220,9 @@ mp_pose = mp.solutions.pose
 
 # For static images:
 with mp_pose.Pose(
-    static_image_mode=True, min_detection_confidence=0.5) as pose:
+    static_image_mode=True,
+    model_complexity=2,
+    min_detection_confidence=0.5) as pose:
   for idx, file in enumerate(file_list):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -213,8 +238,6 @@ with mp_pose.Pose(
     )
     # Draw pose landmarks on the image.
     annotated_image = image.copy()
-    # Use mp_pose.UPPER_BODY_POSE_CONNECTIONS for drawing below when
-    # upper_body_only is set to True.
     mp_drawing.draw_landmarks(
         annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
@@ -258,7 +281,7 @@ and the following usage example.
 
 Supported configuration options:
 
-*   [upperBodyOnly](#upper_body_only)
+*   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
@@ -305,7 +328,7 @@ const pose = new Pose({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`;
 }});
 pose.setOptions({
-  upperBodyOnly: false,
+  modelComplexity: 1,
   smoothLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
@@ -346,16 +369,6 @@ to visualize its associated subgraphs, please see
 *   iOS target:
     [`mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp`](http:/mediapipe/examples/ios/posetrackinggpu/BUILD)
 
-#### Upper-body Only
-
-*   Graph:
-    [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
-*   Android target:
-    [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1uKc6T7KSuA0Mlq2URi5YookHu0U3yoh_/view?usp=sharing)
-    [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu:upperbodyposetrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu/BUILD)
-*   iOS target:
-    [`mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp`](http:/mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD)
-
 ### Desktop
 
 Please first see general instructions for [desktop](../getting_started/cpp.md)
@@ -374,134 +387,6 @@ on how to build MediaPipe examples.
     *   Target:
         [`mediapipe/examples/desktop/pose_tracking:pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/pose_tracking/BUILD)
 
-#### Upper-body Only
-
-*   Running on CPU
-    *   Graph:
-        [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt)
-    *   Target:
-        [`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
-*   Running on GPU
-    *   Graph:
-        [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
-    *   Target:
-        [`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
-
-## Pose Classification and Repetition Counting
-
-One of the applications
-[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
-can enable is fitness. More specifically - pose classification and repetition
-counting. In this section we'll provide basic guidance on building a custom pose
-classifier with the help of a
-[Colab](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
-and wrap it in a simple
-[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
-powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
-are used for demonstration purposes as the most common exercises.
-
-![pose_classification_pushups_and_squats.gif](../images/mobile/pose_classification_pushups_and_squats.gif) |
-:--------------------------------------------------------------------------------------------------------: |
-*Fig 4. Pose classification and repetition counting with MediaPipe Pose.*                                  |
-
-We picked the
-[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
-(k-NN) as the classifier. It's simple and easy to start with. The algorithm
-determines the object's class based on the closest samples in the training set.
-To build it, one needs to:
-
-*   Collect image samples of the target exercises and run pose prediction on
-    them,
-*   Convert obtained pose landmarks to a representation suitable for the k-NN
-    classifier and form a training set,
-*   Perform the classification itself followed by repetition counting.
-
-### Training Set
-
-To build a good classifier appropriate samples should be collected for the
-training set: about a few hundred samples for each terminal state of each
-exercise (e.g., "up" and "down" positions for push-ups). It's important that
-collected samples cover different camera angles, environment conditions, body
-shapes, and exercise variations.
-
-![pose_classification_pushups_un_and_down_samples.jpg](../images/mobile/pose_classification_pushups_un_and_down_samples.jpg) |
-:--------------------------------------------------------------------------------------------------------------------------: |
-*Fig 5. Two terminal states of push-ups.*                                                                                    |
-
-To transform samples into a k-NN classifier training set, either
-[basic](https://drive.google.com/file/d/1z4IM8kG6ipHN6keadjD-F6vMiIIgViKK/view?usp=sharing)
-or
-[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
-Colab could be used. They both use the
-[Python Solution API](#python-solution-api) to run the BlazePose models on given
-images and dump predicted pose landmarks to a CSV file. Additionally, the
-extended Colab provides useful tools to find outliers (e.g., wrongly predicted
-poses) and underrepresented classes (e.g., not covering all camera angles) by
-classifying each sample against the entire training set. After that, you'll be
-able to test the classifier on an arbitrary video right in the Colab.
-
-### Classification
-
-Code of the classifier is available both in the
-[extended](https://drive.google.com/file/d/19txHpN8exWhstO6WVkfmYYVC6uug_oVR/view?usp=sharing)
-Colab and in the
-[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
-Please refer to them for details of the approach described below.
-
-The k-NN algorithm used for pose classification requires a feature vector
-representation of each sample and a metric to compute the distance between two
-such vectors to find the nearest pose samples to a target one.
-
-To convert pose landmarks to a feature vector, we use pairwise distances between
-predefined lists of pose joints, such as distances between wrist and shoulder,
-ankle and hip, and two wrists. Since the algorithm relies on distances, all
-poses are normalized to have the same torso size and vertical torso orientation
-before the conversion.
-
-![pose_classification_pairwise_distances.png](../images/mobile/pose_classification_pairwise_distances.png) |
-:--------------------------------------------------------------------------------------------------------: |
-*Fig 6. Main pairwise distances used for the pose feature vector.*                                         |
-
-To get a better classification result, k-NN search is invoked twice with
-different distance metrics:
-
-*   First, to filter out samples that are almost the same as the target one but
-    have only a few different values in the feature vector (which means
-    differently bent joints and thus other pose class), minimum per-coordinate
-    distance is used as distance metric,
-*   Then average per-coordinate distance is used to find the nearest pose
-    cluster among those from the first search.
-
-Finally, we apply
-[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
-(EMA) smoothing to level any noise from pose prediction or classification. To do
-that, we search not only for the nearest pose cluster, but we calculate a
-probability for each of them and use it for smoothing over time.
-
-### Repetition Counter
-
-To count the repetitions, the algorithm monitors the probability of a target
-pose class. Let's take push-ups with its "up" and "down" terminal states:
-
-*   When the probability of the "down" pose class passes a certain threshold for
-    the first time, the algorithm marks that the "down" pose class is entered.
-*   Once the probability drops below the threshold, the algorithm marks that the
-    "down" pose class has been exited and increases the counter.
-
-To avoid cases when the probability fluctuates around the threshold (e.g., when
-the user pauses between "up" and "down" states) causing phantom counts, the
-threshold used to detect when the state is exited is actually slightly lower
-than the one used to detect when the state is entered. It creates an interval
-where the pose class and the counter can't be changed.
-
-### Future Work
-
-We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
-allow us to use joint angles in the feature vectors, which are more natural and
-easier to configure (although distances can still be useful to detect touches
-between body parts) and to perform rotation normalization of poses and reduce
-the number of camera angles required for accurate k-NN classification.
-
 ## Resources
 
 *   Google AI Blog:
@@ -512,5 +397,6 @@ the number of camera angles required for accurate k-NN classification.
 *   [Models and model cards](./models.md#pose)
 *   [Web demo](https://code.mediapipe.dev/codepen/pose)
 *   [Python Colab](https://mediapipe.page.link/pose_py_colab)
-*   [Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
+
-*   [Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
+[`mAP`]: https://cocodataset.org/#keypoints-eval
+[`[email protected]`]: https\://github.com/cbsudux/Human-Pose-Estimation-101

docs/solutions/pose_classification.md

@@ -0,0 +1,145 @@
+---
+layout: default
+title: Pose Classification
+parent: Pose
+grand_parent: Solutions
+nav_order: 1
+---
+
+# Pose Classification
+{: .no_toc }
+
+<details close markdown="block">
+  <summary>
+    Table of contents
+  </summary>
+  {: .text-delta }
+1. TOC
+{:toc}
+</details>
+---
+
+## Overview
+
+One of the applications
+[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
+can enable is fitness. More specifically - pose classification and repetition
+counting. In this section we'll provide basic guidance on building a custom pose
+classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
+demo within
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
+Push-ups and squats are used for demonstration purposes as the most common
+exercises.
+
+![pose_classification_pushups_and_squats.gif](../images/mobile/pose_classification_pushups_and_squats.gif) |
+:--------------------------------------------------------------------------------------------------------: |
+*Fig 1. Pose classification and repetition counting with MediaPipe Pose.*                                  |
+
+We picked the
+[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
+(k-NN) as the classifier. It's simple and easy to start with. The algorithm
+determines the object's class based on the closest samples in the training set.
+
+**To build it, one needs to:**
+
+1.  Collect image samples of the target exercises and run pose prediction on
+    them,
+2.  Convert obtained pose landmarks to a representation suitable for the k-NN
+    classifier and form a training set using these [Colabs](#colabs),
+3.  Perform the classification itself followed by repetition counting (e.g., in
+    the
+    [ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
+
+## Training Set
+
+To build a good classifier appropriate samples should be collected for the
+training set: about a few hundred samples for each terminal state of each
+exercise (e.g., "up" and "down" positions for push-ups). It's important that
+collected samples cover different camera angles, environment conditions, body
+shapes, and exercise variations.
+
+![pose_classification_pushups_un_and_down_samples.jpg](../images/mobile/pose_classification_pushups_un_and_down_samples.jpg) |
+:--------------------------------------------------------------------------------------------------------------------------: |
+*Fig 2. Two terminal states of push-ups.*                                                                                    |
+
+To transform samples into a k-NN classifier training set, both
+[`Pose Classification Colab (Basic)`] and
+[`Pose Classification Colab (Extended)`] could be used. They use the
+[Python Solution API](./pose.md#python-solution-api) to run the BlazePose models
+on given images and dump predicted pose landmarks to a CSV file. Additionally,
+the [`Pose Classification Colab (Extended)`] provides useful tools to find
+outliers (e.g., wrongly predicted poses) and underrepresented classes (e.g., not
+covering all camera angles) by classifying each sample against the entire
+training set. After that, you'll be able to test the classifier on an arbitrary
+video right in the Colab.
+
+## Classification
+
+Code of the classifier is available both in the
+[`Pose Classification Colab (Extended)`] and in the
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
+Please refer to them for details of the approach described below.
+
+The k-NN algorithm used for pose classification requires a feature vector
+representation of each sample and a metric to compute the distance between two
+such vectors to find the nearest pose samples to a target one.
+
+To convert pose landmarks to a feature vector, we use pairwise distances between
+predefined lists of pose joints, such as distances between wrist and shoulder,
+ankle and hip, and two wrists. Since the algorithm relies on distances, all
+poses are normalized to have the same torso size and vertical torso orientation
+before the conversion.
+
+![pose_classification_pairwise_distances.png](../images/mobile/pose_classification_pairwise_distances.png) |
+:--------------------------------------------------------------------------------------------------------: |
+*Fig 3. Main pairwise distances used for the pose feature vector.*                                         |
+
+To get a better classification result, k-NN search is invoked twice with
+different distance metrics:
+
+*   First, to filter out samples that are almost the same as the target one but
+    have only a few different values in the feature vector (which means
+    differently bent joints and thus other pose class), minimum per-coordinate
+    distance is used as distance metric,
+*   Then average per-coordinate distance is used to find the nearest pose
+    cluster among those from the first search.
+
+Finally, we apply
+[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
+(EMA) smoothing to level any noise from pose prediction or classification. To do
+that, we search not only for the nearest pose cluster, but we calculate a
+probability for each of them and use it for smoothing over time.
+
+## Repetition Counting
+
+To count the repetitions, the algorithm monitors the probability of a target
+pose class. Let's take push-ups with its "up" and "down" terminal states:
+
+*   When the probability of the "down" pose class passes a certain threshold for
+    the first time, the algorithm marks that the "down" pose class is entered.
+*   Once the probability drops below the threshold, the algorithm marks that the
+    "down" pose class has been exited and increases the counter.
+
+To avoid cases when the probability fluctuates around the threshold (e.g., when
+the user pauses between "up" and "down" states) causing phantom counts, the
+threshold used to detect when the state is exited is actually slightly lower
+than the one used to detect when the state is entered. It creates an interval
+where the pose class and the counter can't be changed.
+
+## Future Work
+
+We are actively working on improving
+[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
+prediction. It will allow us to use joint angles in the feature vectors, which
+are more natural and easier to configure (although distances can still be useful
+to detect touches between body parts) and to perform rotation normalization of
+poses and reduce the number of camera angles required for accurate k-NN
+classification.
+
+## Colabs
+
+*   [`Pose Classification Colab (Basic)`]
+*   [`Pose Classification Colab (Extended)`]
+
+[`Pose Classification Colab (Basic)`]: https://mediapipe.page.link/pose_classification_basic
+[`Pose Classification Colab (Extended)`]: https://mediapipe.page.link/pose_classification_extended

docs/solutions/solutions.md

@@ -28,7 +28,7 @@ has_toc: false
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                        | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/tools/tracing_and_profiling.md

@@ -41,6 +41,7 @@ profiler_config {
   trace_enabled: true
   enable_profiler: true
   trace_log_interval_count: 200
+  trace_log_path: "/sdcard/Download/"
 }
 ```
 
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
 
 *   On the desktop, this will be the `/tmp` directory.
 
-*   On Android, this will be the `/sdcard` directory.
+*   On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
 
 *   On iOS, this can be reached through XCode. Select "Window/Devices and
     Simulators" and select the "Devices" tab.
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     *   Include the line below in your `AndroidManifest.xml` file.
 
         ```xml
-        <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+        <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
         ```
 
     *   Grant the permission either upon first app launch, or by going into
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
     events to a trace log files at:
 
     ```bash
-    /sdcard/mediapipe_trace_0.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_0.binarypb
-    /sdcard/mediapipe_trace_1.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_1.binarypb
     ```
 
     After every 5 sec, writing shifts to a successive trace log file, such that
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
     trace files have been written to the device using adb shell.
 
     ```bash
-    adb shell "ls -la /sdcard/"
+    adb shell "ls -la /storage/emulated/0/Download"
     ```
 
-    On android, MediaPipe selects the external storage directory `/sdcard` for
+    On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
     trace logs. This directory can be overridden using the setting
     `trace_log_path`, like:
 
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     profiler_config {
       trace_enabled: true
       enable_profiler: true
-      trace_log_path: "/sdcard/profiles/"
+      trace_log_path: "/sdcard/Download/profiles/"
     }
     ```
 
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
 
     ```bash
     # from your terminal
-    adb pull /sdcard/mediapipe_trace_0.binarypb
+    adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
     # if successful you should see something like
     # /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
     ```

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -16,7 +16,6 @@
     "mediapipe/examples/ios/objectdetectiongpu/BUILD",
     "mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD",
     "mediapipe/examples/ios/posetrackinggpu/BUILD",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD",
     "mediapipe/framework/BUILD",
     "mediapipe/gpu/BUILD",
     "mediapipe/objc/BUILD",
@@ -36,7 +35,6 @@
     "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
     "//mediapipe/examples/ios/objectdetectiontrackinggpu:ObjectDetectionTrackingGpuApp",
     "//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp",
-    "//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
     "//mediapipe/objc:mediapipe_framework_ios"
   ],
   "optionSet" : {
@@ -105,7 +103,6 @@
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectiongpu",
     "mediapipe/examples/ios/posetrackinggpu",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu",
     "mediapipe/framework",
     "mediapipe/framework/deps",
     "mediapipe/framework/formats",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -22,7 +22,6 @@
     "mediapipe/examples/ios/objectdetectiongpu",
     "mediapipe/examples/ios/objectdetectiontrackinggpu",
     "mediapipe/examples/ios/posetrackinggpu",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu",
     "mediapipe/objc"
   ],
   "projectName" : "Mediapipe",

mediapipe/calculators/audio/BUILD

@@ -128,7 +128,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -147,7 +147,7 @@ cc_library(
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp/mfcc",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -168,7 +168,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:resampler",
         "@com_google_audio_tools//audio/dsp:resampler_q",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -208,7 +208,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@com_google_audio_tools//audio/dsp/spectrogram",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -228,7 +228,7 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -242,9 +242,9 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -261,7 +261,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -276,7 +276,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -296,7 +296,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:number_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -314,7 +314,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -333,7 +333,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -352,6 +352,6 @@ cc_test(
         "//mediapipe/framework/tool:validate_type",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:signal_vector_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
@@ -25,7 +25,7 @@ namespace mediapipe {
 
 TEST(AudioDecoderCalculatorTest, TestWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -34,7 +34,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -56,7 +56,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
 
 TEST(AudioDecoderCalculatorTest, Test48KWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -65,7 +65,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -87,7 +87,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
 
 TEST(AudioDecoderCalculatorTest, TestMP3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -96,7 +96,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -118,7 +118,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
 
 TEST(AudioDecoderCalculatorTest, TestAAC) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -127,7 +127,7 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",

mediapipe/calculators/core/BUILD

@@ -233,6 +233,22 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "concatenate_vector_calculator_hdr",
+    hdrs = ["concatenate_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "concatenate_vector_calculator",
     srcs = ["concatenate_vector_calculator.cc"],
@@ -414,7 +430,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -430,7 +446,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -450,6 +466,35 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "non_zero_calculator",
+    srcs = ["non_zero_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "non_zero_calculator_test",
+    size = "small",
+    srcs = ["non_zero_calculator_test.cc"],
+    deps = [
+        ":non_zero_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_test(
     name = "mux_calculator_test",
     srcs = ["mux_calculator_test.cc"],
@@ -651,6 +696,18 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "default_side_packet_calculator",
+    srcs = ["default_side_packet_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "side_packet_to_stream_calculator",
     srcs = ["side_packet_to_stream_calculator.cc"],
@@ -776,7 +833,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -793,7 +850,7 @@ cc_test(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -1024,7 +1081,7 @@ cc_library(
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/memory",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -70,7 +70,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           node {
@@ -90,7 +90,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:ints_plus_one"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));
@@ -197,7 +197,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "force_ints_to_be_timestamp_bound_update"
@@ -229,7 +229,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
             input_stream: "ints_plus_one"
             output_stream: "ints_plus_one_passed_through"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one_passed_through", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
@@ -338,7 +338,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "multiplier"
@@ -362,7 +362,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:multiplied_ints"
           }
-        )");
+        )pb");
     tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -38,14 +38,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
 
 TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {};
@@ -60,14 +60,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {0, 1, 2, 3};
@@ -85,14 +85,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   {
@@ -133,7 +133,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_vector"
         node {
           calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
@@ -143,7 +143,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
             [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("output_vector", &graph_config, &outputs);
@@ -178,7 +178,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         input_side_packet: "max_vec_size"
@@ -186,7 +186,7 @@ TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   // This should override the default of 1 set in the options.
   runner.MutableSidePackets()->Index(0) = Adopt(new int(2));

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -392,7 +392,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         input_stream: "in_3"
@@ -403,7 +403,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
           input_stream: "in_3"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -456,7 +456,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -465,7 +465,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
           input_stream: "in_2"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -505,7 +505,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -519,7 +519,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
             }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -82,7 +82,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -111,7 +111,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
 
 TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -131,7 +131,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -152,7 +152,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:int_packet"
@@ -163,14 +163,14 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -181,7 +181,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }

mediapipe/calculators/core/default_side_packet_calculator.cc

@@ -0,0 +1,104 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kOptionalValueTag[] = "OPTIONAL_VALUE";
+constexpr char kDefaultValueTag[] = "DEFAULT_VALUE";
+constexpr char kValueTag[] = "VALUE";
+
+}  // namespace
+
+// Outputs side packet default value if optional value is not provided.
+//
+// This calculator utilizes the fact that MediaPipe automatically removes
+// optional side packets of the calculator configuration (i.e. OPTIONAL_VALUE).
+// And if it happens - returns default value, otherwise - returns optional
+// value.
+//
+// Input:
+//   OPTIONAL_VALUE (optional) - AnyType (but same type as DEFAULT_VALUE)
+//     Optional side packet value that is outputted by the calculator as is if
+//     provided.
+//
+//   DEFAULT_VALUE - AnyType
+//     Default side pack value that is outputted by the calculator if
+//     OPTIONAL_VALUE is not provided.
+//
+// Output:
+//   VALUE - AnyType (but same type as DEFAULT_VALUE)
+//     Either OPTIONAL_VALUE (if provided) or DEFAULT_VALUE (otherwise).
+//
+// Usage example:
+//   node {
+//     calculator: "DefaultSidePacketCalculator"
+//     input_side_packet: "OPTIONAL_VALUE:segmentation_mask_enabled_optional"
+//     input_side_packet: "DEFAULT_VALUE:segmentation_mask_enabled_default"
+//     output_side_packet: "VALUE:segmentation_mask_enabled"
+//   }
+class DefaultSidePacketCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(DefaultSidePacketCalculator);
+
+absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) {
+  RET_CHECK(cc->InputSidePackets().HasTag(kDefaultValueTag))
+      << "Default value must be provided";
+  cc->InputSidePackets().Tag(kDefaultValueTag).SetAny();
+
+  // Optional input side packet can be unspecified. In this case MediaPipe will
+  // remove it from the calculator config.
+  if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
+    cc->InputSidePackets()
+        .Tag(kOptionalValueTag)
+        .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag))
+        .Optional();
+  }
+
+  RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag));
+  cc->OutputSidePackets().Tag(kValueTag).SetSameAs(
+      &cc->InputSidePackets().Tag(kDefaultValueTag));
+
+  return absl::OkStatus();
+}
+
+absl::Status DefaultSidePacketCalculator::Open(CalculatorContext* cc) {
+  // If optional value is provided it is returned as the calculator output.
+  if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
+    auto& packet = cc->InputSidePackets().Tag(kOptionalValueTag);
+    cc->OutputSidePackets().Tag(kValueTag).Set(packet);
+    return absl::OkStatus();
+  }
+
+  // If no optional value
+  auto& packet = cc->InputSidePackets().Tag(kDefaultValueTag);
+  cc->OutputSidePackets().Tag(kValueTag).Set(packet);
+
+  return absl::OkStatus();
+}
+
+absl::Status DefaultSidePacketCalculator::Process(CalculatorContext* cc) {
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             max_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -51,7 +51,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -61,7 +61,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -76,7 +76,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -86,7 +86,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -101,7 +101,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -111,7 +111,7 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
             min_quantized_value: -2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -57,7 +57,7 @@ namespace mediapipe {
 //
 // The "ALLOW" stream indicates the transition between accepting frames and
 // dropping frames.  "ALLOW = true" indicates the start of accepting frames
-// including the current timestamp, and "ALLOW = true" indicates the start of
+// including the current timestamp, and "ALLOW = false" indicates the start of
 // dropping frames including the current timestamp.
 //
 // FlowLimiterCalculator provides limited support for multiple input streams.

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -126,7 +126,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The LambdaCalculator is used to keep certain frames in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -143,7 +143,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
         input_stream: 'in_1_sampled'
         output_stream: 'out_1'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -271,7 +271,7 @@ REGISTER_CALCULATOR(DropCalculator);
 class FlowLimiterCalculatorTest : public testing::Test {
  protected:
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -296,7 +296,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
         input_stream: 'PACKET:out_1_sampled'
         output_stream: 'PACKET:out_1'
       }
-    )");
+    )pb");
   }
 
   // Parse an absl::Time from RFC3339 format.
@@ -348,10 +348,10 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -419,11 +419,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -483,11 +483,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -548,7 +548,7 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -576,13 +576,13 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -657,7 +657,7 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -685,13 +685,13 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 0
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -64,13 +64,13 @@ const char kMatrixText2[] =
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_side_packet: "MINUEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(
@@ -81,12 +81,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig) {
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_stream: "SUBTRAHEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
@@ -95,12 +95,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "MINUEND:input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);
@@ -124,12 +124,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "SUBTRAHEND:input_matrix"
         input_side_packet: "MINUEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);

mediapipe/calculators/core/merge_calculator_test.cc

@@ -26,31 +26,33 @@ namespace {
 
 // Checks that the calculator fails if no input streams are provided.
 TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     output_stream: "merged_output"
-  )"));
+  )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner.Run().ok());
 }
 
 // Checks that the calculator fails with an incorrect number of output streams.
 TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
-  CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner1(
-    calculator: "MergeCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "input1"
+        calculator: "MergeCalculator"
-    input_stream: "input2"
+        input_stream: "input1"
-  )"));
+        input_stream: "input2"
+      )pb"));
   // Expect calculator to fail.
   EXPECT_FALSE(runner1.Run().ok());
 
-  CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner2(
-    calculator: "MergeCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "input1"
+        calculator: "MergeCalculator"
-    input_stream: "input2"
+        input_stream: "input1"
-    output_stream: "output1"
+        input_stream: "input2"
-    output_stream: "output2"
+        output_stream: "output1"
-  )"));
+        output_stream: "output2"
+      )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner2.Run().ok());
 }
@@ -58,12 +60,12 @@ TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
 // Ensures two streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingTwoStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integers 10, 20, 30, occurring at times 10, 20, 30.
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -102,13 +104,13 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
 // Ensures three streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingThreeStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     input_stream: "input3"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integer 30 occurring at time 30.
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/core/mux_calculator_test.cc

@@ -31,7 +31,7 @@ namespace {
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input stream. All MuxCalculator inputs are present at each timestamp.
-constexpr char kTestGraphConfig1[] = R"proto(
+constexpr char kTestGraphConfig1[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -60,12 +60,12 @@ constexpr char kTestGraphConfig1[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input side packet. All MuxCalculator inputs are present at each
 // timestamp.
-constexpr char kTestGraphConfig2[] = R"proto(
+constexpr char kTestGraphConfig2[] = R"pb(
   input_side_packet: "input_selector"
   input_stream: "input"
   output_stream: "test_output"
@@ -93,12 +93,12 @@ constexpr char kTestGraphConfig2[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with mux input stream handler, and the input selection is driven
 // by an input stream. Only one MuxCalculator input is present at each
 // timestamp.
-constexpr char kTestGraphConfig3[] = R"proto(
+constexpr char kTestGraphConfig3[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -117,7 +117,7 @@ constexpr char kTestGraphConfig3[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 constexpr char kOutputName[] = "test_output";
 constexpr char kInputName[] = "input";
@@ -235,7 +235,7 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
   EXPECT_EQ(output, input_packets);
 }
 
-constexpr char kDualInputGraphConfig[] = R"proto(
+constexpr char kDualInputGraphConfig[] = R"pb(
   input_stream: "input_0"
   input_stream: "input_1"
   input_stream: "input_select"
@@ -247,7 +247,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
   CalculatorGraphConfig config =

mediapipe/calculators/core/non_zero_calculator.cc

@@ -0,0 +1,54 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
+class NonZeroCalculator : public Node {
+ public:
+  static constexpr Input<int>::SideFallback kIn{"INPUT"};
+  static constexpr Output<int>::Optional kOut{"OUTPUT"};
+  static constexpr Output<bool>::Optional kBooleanOut{"OUTPUT_BOOL"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kBooleanOut);
+
+  absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK(kOut(cc).IsConnected() || kBooleanOut(cc).IsConnected())
+        << "At least one output stream is expected.";
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      bool isNonZero = *kIn(cc) != 0;
+      if (kOut(cc).IsConnected()) {
+        kOut(cc).Send(std::make_unique<int>(isNonZero ? 1 : 0));
+      }
+      if (kBooleanOut(cc).IsConnected()) {
+        kBooleanOut(cc).Send(std::make_unique<bool>(isNonZero));
+      }
+    }
+    return absl::OkStatus();
+  }
+};
+
+MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/non_zero_calculator_test.cc

@@ -0,0 +1,93 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+class NonZeroCalculatorTest : public ::testing::Test {
+ protected:
+  NonZeroCalculatorTest()
+      : runner_(
+            R"pb(
+              calculator: "NonZeroCalculator"
+              input_stream: "INPUT:input"
+              output_stream: "OUTPUT:output"
+              output_stream: "OUTPUT_BOOL:output_bool"
+            )pb") {}
+
+  void SetInput(const std::vector<int>& inputs) {
+    int timestamp = 0;
+    for (const auto input : inputs) {
+      runner_.MutableInputs()
+          ->Get("INPUT", 0)
+          .packets.push_back(MakePacket<int>(input).At(Timestamp(timestamp++)));
+    }
+  }
+
+  std::vector<int> GetOutput() {
+    std::vector<int> result;
+    for (const auto output : runner_.Outputs().Get("OUTPUT", 0).packets) {
+      result.push_back(output.Get<int>());
+    }
+    return result;
+  }
+
+  std::vector<bool> GetOutputBool() {
+    std::vector<bool> result;
+    for (const auto output : runner_.Outputs().Get("OUTPUT_BOOL", 0).packets) {
+      result.push_back(output.Get<bool>());
+    }
+    return result;
+  }
+
+  CalculatorRunner runner_;
+};
+
+TEST_F(NonZeroCalculatorTest, ProducesZeroOutputForZeroInput) {
+  SetInput({0});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(0));
+  EXPECT_THAT(GetOutputBool(), ::testing::ElementsAre(false));
+}
+
+TEST_F(NonZeroCalculatorTest, ProducesNonZeroOutputForNonZeroInput) {
+  SetInput({1, 2, 3, -4, 5});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 1, 1, 1, 1));
+  EXPECT_THAT(GetOutputBool(),
+              ::testing::ElementsAre(true, true, true, true, true));
+}
+
+TEST_F(NonZeroCalculatorTest, SwitchesBetweenNonZeroAndZeroOutput) {
+  SetInput({1, 0, 3, 0, 5});
+  MP_ASSERT_OK(runner_.Run());
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 0, 1, 0, 1));
+  EXPECT_THAT(GetOutputBool(),
+              ::testing::ElementsAre(true, false, true, false, true));
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_presence_calculator_test.cc

@@ -40,7 +40,7 @@ MATCHER_P2(BoolPacket, value, timestamp, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> output_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'allow'
         input_stream: 'value'
         node {
@@ -54,7 +54,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'PACKET:gated_value'
           output_stream: 'PRESENCE:presence'
         }
-      )");
+      )pb");
   tool::AddVectorSink("presence", &graph_config, &output_packets);
 
   CalculatorGraph graph;

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
-  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     output_data_id_ = cc->Outputs().GetId("", 0);
   }
 
-  period_count_ = 0;
   frame_rate_ = resampler_options.frame_rate();
-  base_timestamp_ = resampler_options.has_base_timestamp()
-                        ? Timestamp(resampler_options.base_timestamp())
-                        : Timestamp::Unset();
   start_time_ = resampler_options.has_start_time()
                     ? Timestamp(resampler_options.start_time())
                     : Timestamp::Min();
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     }
   }
 
-  if (jitter_ != 0.0) {
+  strategy_ = GetSamplingStrategy(resampler_options);
-    if (resampler_options.output_header() !=
+
-        PacketResamplerCalculatorOptions::NONE) {
+  return strategy_->Open(cc);
-      LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
-                      "the actual value.";
-    }
-    if (flush_last_packet_) {
-      flush_last_packet_ = false;
-      LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
-                      "ignored, because we are adding jitter.";
-    }
-    const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
-    random_ = CreateSecureRandom(seed);
-    if (random_ == nullptr) {
-      return absl::Status(
-          absl::StatusCode::kInvalidArgument,
-          "SecureRandom is not available.  With \"jitter\" specified, "
-          "PacketResamplerCalculator processing cannot proceed.");
-    }
-    packet_reservoir_random_ = CreateSecureRandom(seed);
-  }
-  packet_reservoir_ =
-      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
-  return absl::OkStatus();
 }
 
 absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
       return absl::OkStatus();
     }
   }
-  if (jitter_ != 0.0 && random_ != nullptr) {
+
-    // Packet reservior is used to make sure there's an output for every period,
+  if (absl::Status status = strategy_->Process(cc); !status.ok()) {
-    // e.g. partial period at the end of the stream.
+    return status;  // Avoid MP_RETURN_IF_ERROR macro for external release.
-    if (packet_reservoir_->IsEnabled() &&
-        (first_timestamp_ == Timestamp::Unset() ||
-         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
-      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
-      packet_reservoir_->AddSample(curr_packet);
-    }
-    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
-  } else {
-    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
   }
+
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
-  return absl::OkStatus();
-}
 
-void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
-  next_output_timestamp_min_ = first_timestamp_;
-  if (jitter_with_reflection_) {
-    next_output_timestamp_ =
-        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
-    return;
-  }
-  next_output_timestamp_ =
-      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
-}
-
-void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
-  packet_reservoir_->Clear();
-  if (jitter_with_reflection_) {
-    next_output_timestamp_min_ += frame_time_usec_;
-    Timestamp next_output_timestamp_max_ =
-        next_output_timestamp_min_ + frame_time_usec_;
-
-    next_output_timestamp_ += frame_time_usec_ +
-                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
-                              jitter_usec_;
-    next_output_timestamp_ = Timestamp(ReflectBetween(
-        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
-        next_output_timestamp_max_.Value()));
-    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
-    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
-    return;
-  }
-  packet_reservoir_->Disable();
-  next_output_timestamp_ +=
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_NE(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    first_timestamp_ = cc->InputTimestamp();
-    InitializeNextOutputTimestampWithJitter();
-    if (first_timestamp_ == next_output_timestamp_) {
-      OutputWithinLimits(
-          cc,
-          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
-      UpdateNextOutputTimestampWithJitter();
-    }
-    return absl::OkStatus();
-  }
-
-  if (frame_time_usec_ <
-      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
-    LOG_FIRST_N(WARNING, 2)
-        << "Adding jitter is not very useful when upsampling.";
-  }
-
-  while (true) {
-    const int64 last_diff =
-        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-    RET_CHECK_GT(last_diff, 0);
-    const int64 curr_diff =
-        (next_output_timestamp_ - cc->InputTimestamp()).Value();
-    if (curr_diff > 0) {
-      break;
-    }
-    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
-                                ? last_packet_
-                                : cc->Inputs().Get(input_data_id_).Value())
-                               .At(next_output_timestamp_));
-    UpdateNextOutputTimestampWithJitter();
-    // From now on every time a packet is emitted the timestamp of the next
-    // packet becomes known; that timestamp is stored in next_output_timestamp_.
-    // The only exception to this rule is the packet emitted from Close() which
-    // can only happen when jitter_with_reflection is enabled but in this case
-    // next_output_timestamp_min_ is a non-decreasing lower bound of any
-    // subsequent packet.
-    const Timestamp timestamp_bound = jitter_with_reflection_
-                                          ? next_output_timestamp_min_
-                                          : next_output_timestamp_;
-    cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
-  }
-  return absl::OkStatus();
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_EQ(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    // This is the first packet, initialize the first_timestamp_.
-    if (base_timestamp_ == Timestamp::Unset()) {
-      // Initialize first_timestamp_ with exactly the first packet timestamp.
-      first_timestamp_ = cc->InputTimestamp();
-    } else {
-      // Initialize first_timestamp_ with the first packet timestamp
-      // aligned to the base_timestamp_.
-      int64 first_index = MathUtil::SafeRound<int64, double>(
-          (cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
-      first_timestamp_ =
-          base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
-    }
-    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
-      cc->Outputs()
-          .Tag("VIDEO_HEADER")
-          .Add(new VideoHeader(video_header_), Timestamp::PreStream());
-    }
-  }
-  const Timestamp received_timestamp = cc->InputTimestamp();
-  const int64 received_timestamp_idx =
-      TimestampToPeriodIndex(received_timestamp);
-  // Only consider the received packet if it belongs to the current period
-  // (== period_count_) or to a newer one (> period_count_).
-  if (received_timestamp_idx >= period_count_) {
-    // Fill the empty periods until we are in the same index as the received
-    // packet.
-    while (received_timestamp_idx > period_count_) {
-      OutputWithinLimits(
-          cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
-      ++period_count_;
-    }
-    // Now, if the received packet has a timestamp larger than the middle of
-    // the current period, we can send a packet without waiting. We send the
-    // one closer to the middle.
-    Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
-    if (received_timestamp >= target_timestamp) {
-      bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
-      bool send_current =
-          !have_last_packet || (received_timestamp - target_timestamp <=
-                                target_timestamp - last_packet_.Timestamp());
-      if (send_current) {
-        OutputWithinLimits(
-            cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
-      } else {
-        OutputWithinLimits(cc, last_packet_.At(target_timestamp));
-      }
-      ++period_count_;
-    }
-    // TODO: Add a mechanism to the framework to allow these packets
-    // to be output earlier (without waiting for a much later packet to
-    // arrive)
-
-    // Update the bound for the next packet.
-    cc->Outputs()
-        .Get(output_data_id_)
-        .SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
-  }
   return absl::OkStatus();
 }
 
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
   if (!cc->GraphStatus().ok()) {
     return absl::OkStatus();
   }
-  // Emit the last packet received if we have at least one packet, but
+
-  // haven't sent anything for its period.
+  return strategy_->Close(cc);
-  if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
+}
-      TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
+
-    OutputWithinLimits(cc,
+std::unique_ptr<PacketResamplerStrategy>
-                       last_packet_.At(PeriodIndexToTimestamp(period_count_)));
+PacketResamplerCalculator::GetSamplingStrategy(
+    const PacketResamplerCalculatorOptions& options) {
+  if (options.reproducible_sampling()) {
+    if (!options.jitter_with_reflection()) {
+      LOG(WARNING)
+          << "reproducible_sampling enabled w/ jitter_with_reflection "
+             "disabled. "
+          << "reproducible_sampling always uses jitter with reflection, "
+          << "Ignoring jitter_with_reflection setting.";
+    }
+    return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
   }
-  if (!packet_reservoir_->IsEmpty()) {
+
-    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  if (options.jitter() == 0) {
+    return absl::make_unique<NoJitterStrategy>(this);
   }
-  return absl::OkStatus();
+
+  if (options.jitter_with_reflection()) {
+    return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
+  }
+
+  // With jitter and no reflection.
+  return absl::make_unique<JitterWithoutReflectionStrategy>(this);
 }
 
 Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
   }
 }
 
+absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    LOG(INFO) << "Emitting pack from reservoir.";
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (packet_reservoir_->IsEnabled() &&
+      (first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (first_timestamp_ == Timestamp::Unset()) {
+    first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestampWithJitter();
+    if (first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
+    // From now on every time a packet is emitted the timestamp of the next
+    // packet becomes known; that timestamp is stored in next_output_timestamp_.
+    // The only exception to this rule is the packet emitted from Close() which
+    // can only happen when jitter_with_reflection is enabled but in this case
+    // next_output_timestamp_min_ is a non-decreasing lower bound of any
+    // subsequent packet.
+    const Timestamp timestamp_bound = next_output_timestamp_min_;
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(timestamp_bound);
+  }
+  return absl::OkStatus();
+}
+
+void LegacyJitterWithReflectionStrategy::
+    InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
+  next_output_timestamp_ =
+      first_timestamp_ +
+      random_->UnbiasedUniform64(calculator_->frame_time_usec_);
+}
+
+void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  next_output_timestamp_min_ += calculator_->frame_time_usec_;
+  Timestamp next_output_timestamp_max_ =
+      next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+  next_output_timestamp_ +=
+      calculator_->frame_time_usec_ +
+      random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
+      calculator_->jitter_usec_;
+  next_output_timestamp_ = Timestamp(ReflectBetween(
+      next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+      next_output_timestamp_max_.Value()));
+  CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+  CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+}
+
+absl::Status ReproducibleJitterWithReflectionStrategy::Open(
+    CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Close(
+    CalculatorContext* cc) {
+  // If last packet is non-empty and a packet hasn't been emitted for this
+  // period, emit the last packet.
+  if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(next_output_timestamp_));
+  }
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+
+  if (calculator_->last_packet_.IsEmpty()) {
+    // last_packet is empty, this is the first packet of the stream.
+
+    InitializeNextOutputTimestamp(current_packet.Timestamp());
+
+    // If next_output_timestamp_ happens to fall before current_packet, emit
+    // current packet.  Only a single packet can be emitted at the beginning
+    // of the stream.
+    if (next_output_timestamp_ < current_packet.Timestamp()) {
+      calculator_->OutputWithinLimits(
+          cc, current_packet.At(next_output_timestamp_));
+      packet_emitted_this_period_ = true;
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Last packet is set, so we are mid-stream.
+  if (calculator_->frame_time_usec_ <
+      (current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
+          .Value()) {
+    // Note, if the stream is upsampling, this could lead to the same packet
+    // being emitted twice.  Upsampling and jitter doesn't make much sense
+    // but does technically work.
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  // Since we may be upsampling, we need to iteratively advance the
+  // next_output_timestamp_ one period at a time until it reaches the period
+  // current_packet is in.  During this process, last_packet and/or
+  // current_packet may be repeatly emitted.
+
+  UpdateNextOutputTimestamp(current_packet.Timestamp());
+
+  while (!packet_emitted_this_period_ &&
+         next_output_timestamp_ <= current_packet.Timestamp()) {
+    // last_packet < next_output_timestamp_ <= current_packet,
+    // so emit the closest packet.
+    Packet packet_to_emit =
+        current_packet.Timestamp() - next_output_timestamp_ <
+                next_output_timestamp_ - calculator_->last_packet_.Timestamp()
+            ? current_packet
+            : calculator_->last_packet_;
+    calculator_->OutputWithinLimits(cc,
+                                    packet_to_emit.At(next_output_timestamp_));
+
+    packet_emitted_this_period_ = true;
+
+    // If we are upsampling, packet_emitted_this_period_ can be reset by
+    // the following UpdateNext and the loop will iterate.
+    UpdateNextOutputTimestamp(current_packet.Timestamp());
+  }
+
+  // Set the bounds on the output stream.  Note, if we emitted a packet
+  // above, it will already be set at next_output_timestamp_ + 1, in which
+  // case we have to skip setting it.
+  if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
+      next_output_timestamp_) {
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (next_output_timestamp_min_ != Timestamp::Unset()) {
+    return;
+  }
+
+  next_output_timestamp_min_ = Timestamp(0);
+  next_output_timestamp_ =
+      Timestamp(GetNextRandom(calculator_->frame_time_usec_));
+
+  // While the current timestamp is ahead of the max (i.e. min + frame_time),
+  // fast-forward.
+  while (current_timestamp >=
+         next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    packet_emitted_this_period_ = true;  // Force update...
+    UpdateNextOutputTimestamp(current_timestamp);
+  }
+}
+
+void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (packet_emitted_this_period_ &&
+      current_timestamp >=
+          next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    next_output_timestamp_min_ += calculator_->frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+    next_output_timestamp_ += calculator_->frame_time_usec_ +
+                              GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
+                              calculator_->jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+
+    packet_emitted_this_period_ = false;
+  }
+}
+
+absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  // Packet reservior is used to make sure there's an output for every period,
+  // e.g. partial period at the end of the stream.
+  if (packet_reservoir_->IsEnabled() &&
+      (calculator_->first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    calculator_->first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestamp();
+    if (calculator_->first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestamp();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestamp();
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
+  next_output_timestamp_min_ = calculator_->first_timestamp_;
+  next_output_timestamp_ = calculator_->first_timestamp_ +
+                           calculator_->frame_time_usec_ * random_->RandFloat();
+}
+
+void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
+  packet_reservoir_->Clear();
+  packet_reservoir_->Disable();
+  next_output_timestamp_ += calculator_->frame_time_usec_ *
+                            ((1.0 - calculator_->jitter_) +
+                             2.0 * calculator_->jitter_ * random_->RandFloat());
+}
+
+absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+  base_timestamp_ = resampler_options.has_base_timestamp()
+                        ? Timestamp(resampler_options.base_timestamp())
+                        : Timestamp::Unset();
+
+  period_count_ = 0;
+
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
+  // Emit the last packet received if we have at least one packet, but
+  // haven't sent anything for its period.
+  if (calculator_->first_timestamp_ != Timestamp::Unset() &&
+      calculator_->flush_last_packet_ &&
+      calculator_->TimestampToPeriodIndex(
+          calculator_->last_packet_.Timestamp()) == period_count_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(
+                calculator_->PeriodIndexToTimestamp(period_count_)));
+  }
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    // This is the first packet, initialize the first_timestamp_.
+    if (base_timestamp_ == Timestamp::Unset()) {
+      // Initialize first_timestamp_ with exactly the first packet timestamp.
+      calculator_->first_timestamp_ = cc->InputTimestamp();
+    } else {
+      // Initialize first_timestamp_ with the first packet timestamp
+      // aligned to the base_timestamp_.
+      int64 first_index = MathUtil::SafeRound<int64, double>(
+          (cc->InputTimestamp() - base_timestamp_).Seconds() *
+          calculator_->frame_rate_);
+      calculator_->first_timestamp_ =
+          base_timestamp_ +
+          TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
+    }
+    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
+      cc->Outputs()
+          .Tag("VIDEO_HEADER")
+          .Add(new VideoHeader(calculator_->video_header_),
+               Timestamp::PreStream());
+    }
+  }
+  const Timestamp received_timestamp = cc->InputTimestamp();
+  const int64 received_timestamp_idx =
+      calculator_->TimestampToPeriodIndex(received_timestamp);
+  // Only consider the received packet if it belongs to the current period
+  // (== period_count_) or to a newer one (> period_count_).
+  if (received_timestamp_idx >= period_count_) {
+    // Fill the empty periods until we are in the same index as the received
+    // packet.
+    while (received_timestamp_idx > period_count_) {
+      calculator_->OutputWithinLimits(
+          cc, calculator_->last_packet_.At(
+                  calculator_->PeriodIndexToTimestamp(period_count_)));
+      ++period_count_;
+    }
+    // Now, if the received packet has a timestamp larger than the middle of
+    // the current period, we can send a packet without waiting. We send the
+    // one closer to the middle.
+    Timestamp target_timestamp =
+        calculator_->PeriodIndexToTimestamp(period_count_);
+    if (received_timestamp >= target_timestamp) {
+      bool have_last_packet =
+          (calculator_->last_packet_.Timestamp() != Timestamp::Unset());
+      bool send_current =
+          !have_last_packet ||
+          (received_timestamp - target_timestamp <=
+           target_timestamp - calculator_->last_packet_.Timestamp());
+      if (send_current) {
+        calculator_->OutputWithinLimits(cc,
+                                        cc->Inputs()
+                                            .Get(calculator_->input_data_id_)
+                                            .Value()
+                                            .At(target_timestamp));
+      } else {
+        calculator_->OutputWithinLimits(
+            cc, calculator_->last_packet_.At(target_timestamp));
+      }
+      ++period_count_;
+    }
+    // TODO: Add a mechanism to the framework to allow these packets
+    // to be output earlier (without waiting for a much later packet to
+    // arrive)
+
+    // Update the bound for the next packet.
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(
+            calculator_->PeriodIndexToTimestamp(period_count_));
+  }
+  return absl::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -55,7 +55,7 @@ class PacketReservoir {
 //     correspond to timestamp t.
 //   - The next packet is chosen randomly (uniform distribution) among frames
 //     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
-//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//     - if jitter_with_reflection is true, the timestamp will be reflected
 //       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
 //       so that its marginal distribution is uniform within this interval.
 //       In the formula, t_0 is the timestamp of the first sampled
@@ -66,6 +66,17 @@ class PacketReservoir {
 //     the resampling.  For Cloud ML Video Intelligence API, the hash of the
 //     input video should serve this purpose.  For YouTube, either video ID or
 //     content hex ID of the input video should do.
+//   - If reproducible_samping is true, care is taken to allow reproducible
+//     "mid-stream" sampling.  The calculator can be executed on a stream that
+//     doesn't start at the first period.  For instance, if the calculator
+//     is run on a 10 second stream it will produce the same set of samples
+//     as two runs of the calculator, the first with 3 seconds of input starting
+//     at time 0 and the second with 7 seconds of input starting at time +3s.
+//     - In order to guarantee the exact same samples, 1) the inputs must be
+//       aligned with the sampling period.  For instance, if the sampling rate
+//       is 2 frames per second, streams should be aligned on 0.5 second
+//       boundaries, and 2) the stream must include at least one extra packet
+//       before and after the second aligned sampling period.
 //
 // If jitter_ is not specified:
 //   - The first packet defines the first_timestamp of the output stream,
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
   absl::Status Process(CalculatorContext* cc) override;
 
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  absl::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  absl::Status ProcessWithoutJitter(CalculatorContext* cc);
-
   // Given the current count of periods that have passed, this returns
   // the next valid timestamp of the middle point of the next period:
   //    if count is 0, it returns the first_timestamp_.
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Outputs a packet if it is in range (start_time_, end_time_).
   void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
 
+ protected:
+  // Returns Sampling Strategy to use.
+  //
+  // Virtual to allow injection of testing strategies.
+  virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& options);
+
+ private:
+  std::unique_ptr<class PacketResamplerStrategy> strategy_;
+
   // The timestamp of the first packet received.
   Timestamp first_timestamp_;
 
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Inverse of frame_rate_.
   int64 frame_time_usec_;
 
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
   VideoHeader video_header_;
   // The "DATA" input stream.
   CollectionItemId input_data_id_;
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
   CollectionItemId output_data_id_;
 
   // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  // than the final stream timestamp.
   bool flush_last_packet_;
 
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
   double jitter_ = 0.0;
-  bool jitter_with_reflection_;
-  int64 jitter_usec_;
-  Timestamp next_output_timestamp_;
-  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
-  // kept within the interval
-  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
-  Timestamp next_output_timestamp_min_;
 
-  // If specified, output timestamps are aligned with base_timestamp.
+  int64 jitter_usec_;
-  // Otherwise, they are aligned with the first input timestamp.
+
-  Timestamp base_timestamp_;
+  // The last packet that was received.
+  Packet last_packet_;
 
   // If specified, only outputs at/after start_time are included.
   Timestamp start_time_;
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
 
   // If set, the output timestamps nearest to start_time and end_time
   // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.W
+  // between start_time and end_time.
   bool round_limits_;
 
+  // Allow strategies access to all internal calculator state.
+  //
+  // The calculator and strategies are intimiately tied together so this should
+  // not break encapsulation.
+  friend class LegacyJitterWithReflectionStrategy;
+  friend class ReproducibleJitterWithReflectionStrategy;
+  friend class JitterWithoutReflectionStrategy;
+  friend class NoJitterStrategy;
+};
+
+// Abstract class encapsulating sampling stategy.
+//
+// These are used solely by PacketResamplerCalculator, but are exposed here
+// to facilitate tests.
+class PacketResamplerStrategy {
+ public:
+  PacketResamplerStrategy(PacketResamplerCalculator* calculator)
+      : calculator_(calculator) {}
+  virtual ~PacketResamplerStrategy() = default;
+
+  // Delegate for CalculatorBase::Open.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Open(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Close.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Close(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Process.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Process(CalculatorContext* cc) = 0;
+
+ protected:
+  // Calculator running strategy.
+  PacketResamplerCalculator* calculator_;
+};
+
+// Strategy that applies Jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+//
+// This applies the legacy jitter with reflection which doesn't allow
+// for reproducibility of sampling when starting mid-stream.  This is maintained
+// for backward compatibility.
+class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void InitializeNextOutputTimestampWithJitter();
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
   // packet reservior used for sampling random packet out of partial
   // period when jitter is enabled
   std::unique_ptr<PacketReservoir> packet_reservoir_;
+
   // random number generator used in packet_reservior_.
   std::unique_ptr<RandomBase> packet_reservoir_random_;
 };
 
+// Strategy that applies reproducible jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+class ReproducibleJitterWithReflectionStrategy
+    : public PacketResamplerStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategy(
+      PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ protected:
+  // Returns next random in range (0,n].
+  //
+  // Exposed as virtual function for testing Jitter with reflection.
+  // This is the only way random_ is accessed.
+  virtual uint64 GetNextRandom(uint64 n) {
+    return random_->UnbiasedUniform64(n);
+  }
+
+ private:
+  // Initializes Jitter with reflection.
+  //
+  // This will fast-forward to the period containing current_timestamp.
+  // next_output_timestamp_ is guarnateed to be current_timestamp's period
+  // and packet_emitted_this_period_ will be set to false.
+  void InitializeNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Potentially advances next_output_timestamp_ a single period.
+  //
+  // next_output_timestamp_ will only be advanced if packet_emitted_this_period_
+  // is false.  next_output_timestamp_ will never be advanced beyond
+  // current_timestamp's period.
+  //
+  // However, next_output_timestamp_ could fall before current_timestamp's
+  // period since only a single period can be advanced at a time.
+  void UpdateNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // Indicates packet was emitted for current period (i.e. the period
+  // next_output_timestamp_ falls in.
+  bool packet_emitted_this_period_ = false;
+};
+
+// Strategy that applies Jitter without reflection based sampling.
+//
+// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
+// not enabled.
+class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestamp();
+
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestamp();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // packet reservior used for sampling random packet out of partial period.
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+// Strategy that applies sampling without any jitter.
+//
+// Used by PacketResamplerCalculator when jitter is not enabled.
+class NoJitterStrategy : public PacketResamplerStrategy {
+ public:
+  NoJitterStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Number of periods that have passed (= #packets sent to the output).
+  int64 period_count_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+};
+
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+
+  // Enables reflection when applying jitter.
+  //
+  // This option is ignored when reproducible_sampling is true, in which case
+  // reflection will be used.
+  //
+  // New use cases should use reproducible_sampling = true, as
+  // jitter_with_reflection is deprecated and will be removed at some point.
   optional bool jitter_with_reflection = 9 [default = false];
 
+  // If set, enabled reproducible sampling, allowing frames to be sampled
+  // without regards to where the stream starts.  See
+  // packet_resampler_calculator.h for details.
+  //
+  // This enables reflection (ignoring jitter_with_reflection setting).
+  optional bool reproducible_sampling = 10 [default = false];
+
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.
   //

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -30,6 +30,7 @@
 
 namespace mediapipe {
 
+using ::testing::ElementsAre;
 namespace {
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
   static int static_count_;
 };
 
+// Matcher for Packets with uint64 payload, comparing arg packet's
+// timestamp and uint64 payload.
+MATCHER_P2(PacketAtTimestamp, payload, timestamp,
+           absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
+                        payload, " @ time ", timestamp)) {
+  if (timestamp != arg.Timestamp().Value()) {
+    *result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
+    return false;
+  }
+  int64 actual_payload = arg.template Get<int64>();
+  if (actual_payload != payload) {
+    *result_listener << "with incorrect payload = " << actual_payload;
+    return false;
+  }
+  return true;
+}
+
+// JitterWithReflectionStrategy child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleJitterWithReflectionStrategyForTesting
+    : public ReproducibleJitterWithReflectionStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategyForTesting(
+      PacketResamplerCalculator* calculator)
+      : ReproducibleJitterWithReflectionStrategy(calculator) {}
+
+  // Statically accessed random sequence to use for jitter with reflection.
+  //
+  // An EXPECT will fail if sequence is less than the number requested during
+  // processing.
+  static std::vector<uint64> random_sequence;
+
+ protected:
+  virtual uint64 GetNextRandom(uint64 n) {
+    EXPECT_LT(sequence_index_, random_sequence.size());
+    return random_sequence[sequence_index_++] % n;
+  }
+
+ private:
+  int32 sequence_index_ = 0;
+};
+std::vector<uint64>
+    ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
+
+// PacketResamplerCalculator child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleResamplerCalculatorForTesting
+    : public PacketResamplerCalculator {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    return PacketResamplerCalculator::GetContract(cc);
+  }
+
+ protected:
+  std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& Options) {
+    return absl::make_unique<
+        ReproducibleJitterWithReflectionStrategyForTesting>(this);
+  }
+};
+
+REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
+
 int SimpleRunner::static_count_ = 0;
 
 TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
@@ -380,7 +452,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
 }
 
 TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "PacketResamplerCalculator"
     input_stream: "DATA:in_data"
     input_stream: "VIDEO_HEADER:in_video_header"
@@ -389,7 +461,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
     options {
       [mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 }
     }
-  )"));
+  )pb"));
 
   for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
     runner.MutableInputs()->Tag("DATA").packets.push_back(
@@ -633,7 +705,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
 
 TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "PacketResamplerCalculator"
         input_side_packet: "OPTIONS:options"
         input_stream: "input"
@@ -643,16 +715,16 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
             frame_rate: 60
             base_timestamp: 0
           }
-        })");
+        })pb");
 
   {
     SimpleRunner runner(node_config);
     auto options =
         new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(
-            R"(
+            R"pb(
               [mediapipe.PacketResamplerCalculatorOptions.ext] {
                 frame_rate: 30
-              })"));
+              })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
     MP_ASSERT_OK(runner.Run());
@@ -662,12 +734,12 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
     SimpleRunner runner(node_config);
 
     auto options =
-        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"(
+        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"pb(
           merge_fields: false
           [mediapipe.PacketResamplerCalculatorOptions.ext] {
             frame_rate: 30
             base_timestamp: 0
-          })"));
+          })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -69,7 +69,7 @@ MATCHER_P2(PairPacket, timestamp, pair, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> in_prev;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -93,7 +93,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'previous2'
           output_stream: 'pair'
         }
-      )");
+      )pb");
   tool::AddVectorSink("pair", &graph_config_, &in_prev);
 
   CalculatorGraph graph_;
@@ -169,7 +169,7 @@ REGISTER_CALCULATOR(PacketOnCloseCalculator);
 TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -192,7 +192,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -231,7 +231,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -253,7 +253,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -278,7 +278,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -300,7 +300,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -331,7 +331,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
 TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -354,7 +354,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -386,7 +386,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           input_stream: 'force_main_empty'
           input_stream: 'force_loop_empty'
@@ -424,7 +424,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
@@ -724,7 +724,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           # Drops "loop" when set to "true", delaying output of prev_loop, hence
           # delaying output of the graph.
@@ -755,7 +755,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -53,7 +53,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -63,7 +63,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -80,7 +80,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -90,7 +90,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -107,7 +107,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -117,7 +117,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
             min_quantized_value: -1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -133,7 +133,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -143,7 +143,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
   runner.MutableInputs()
@@ -171,7 +171,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -181,7 +181,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {-65.0f, 65.0f};
   runner.MutableInputs()

mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc

@@ -70,13 +70,13 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
 constexpr int kNumImageFrames = 5;
 constexpr int kNumFinished = 3;
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "RealTimeFlowLimiterCalculator"
     input_stream: "raw_frames"
     input_stream: "FINISHED:finished"
     input_stream_info: { tag_index: "FINISHED" back_edge: true }
     output_stream: "gated_frames"
-  )");
+  )pb");
 }
 
 // Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
@@ -219,7 +219,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The two LambdaCalculators are used to keep certain packet sets in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       input_stream: 'in_2'
       node {
@@ -256,7 +256,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
         output_stream: 'out_1'
         output_stream: 'out_2'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -344,7 +344,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
   std::vector<Packet> a_passed;
   std::vector<Packet> b_passed;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_a'
         input_stream: 'in_b'
         input_stream: 'finished'
@@ -360,7 +360,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
           output_stream: 'in_b_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
   tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
@@ -442,7 +442,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
 TEST(RealTimeFlowLimiterCalculator, CanConsume) {
   std::vector<Packet> in_sampled_packets_;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         input_stream: 'finished'
         node {
@@ -455,7 +455,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
           output_stream: 'in_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
   tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);

mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc

@@ -36,7 +36,7 @@ using testing::HasSubstr;
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -45,7 +45,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -58,7 +58,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -67,7 +67,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -79,7 +79,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -88,7 +88,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
               input_side_packet: "side_packet"
               output_stream: "DOES_NOT_EXIST:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -102,7 +102,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -113,7 +113,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
               output_stream: "AT_TICK:packet0"
               output_stream: "AT_PRE_STREAM:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -127,7 +127,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -136,7 +136,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
               output_stream: "AT_PRESTREAM:0:packet0"
               output_stream: "AT_PRESTREAM:1:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -149,7 +149,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -158,7 +158,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
               input_side_packet: "side_packet1"
               output_stream: "AT_PRESTREAM:packet0"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -209,7 +209,7 @@ TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
 TEST(SidePacketToStreamCalculator, AtTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -219,7 +219,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -251,7 +251,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
 TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -265,7 +265,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
               output_stream: "AT_TICK:0:packet0"
               output_stream: "AT_TICK:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;
@@ -305,7 +305,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
 TEST(SidePacketToStreamCalculator, AtTimestamp) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -315,7 +315,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -337,7 +337,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
 TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -350,7 +350,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
               output_stream: "AT_TIMESTAMP:0:packet0"
               output_stream: "AT_TIMESTAMP:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -122,7 +122,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -138,7 +138,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -171,7 +171,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -183,7 +183,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -196,7 +196,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -209,7 +209,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -223,7 +223,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -238,7 +238,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -252,7 +252,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -266,7 +266,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -281,7 +281,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -298,7 +298,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -334,7 +334,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -349,7 +349,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -377,7 +377,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -394,7 +394,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -163,7 +163,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -179,7 +179,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -214,7 +214,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -226,7 +226,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -240,7 +240,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -253,7 +253,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -269,7 +269,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -284,7 +284,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -299,7 +299,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -313,7 +313,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -330,7 +330,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -347,7 +347,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -385,7 +385,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -400,7 +400,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -428,7 +428,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -445,7 +445,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -511,7 +511,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -524,7 +524,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -536,7 +536,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -552,7 +552,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -592,7 +592,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -609,7 +609,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -646,7 +646,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -661,7 +661,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);

mediapipe/calculators/image/BUILD

@@ -410,7 +410,9 @@ cc_library(
     srcs = ["image_properties_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -285,7 +285,7 @@ absl::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
 
   // Run cropping shader on GPU.
   {
-    gpu_helper_.BindFramebuffer(dst_tex);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(dst_tex);
 
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(src_tex.target(), src_tex.name());

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -41,7 +41,7 @@ constexpr char kWidthTag[] = "WIDTH";
 TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -54,7 +54,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -79,7 +79,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -94,7 +94,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -119,7 +119,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
 TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "WIDTH:crop_width"
@@ -136,7 +136,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -168,7 +168,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "RECT:rect"
@@ -184,7 +184,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -196,9 +196,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
       calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
   auto& inputs = cc->Inputs();
   mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
-      R"(
+      R"pb(
         width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
-      )");
+      )pb");
   inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
   RectSpec expectRect = {
       .width = 1,

mediapipe/calculators/image/image_file_properties_calculator_test.cc

@@ -50,11 +50,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_stream: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -79,11 +79,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_side_packet: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Index(0) =
@@ -108,11 +108,11 @@ TEST(ImageFilePropertiesCalculatorTest,
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/image/image_properties_calculator.cc

@@ -12,25 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/image_frame.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-namespace {
-constexpr char kImageFrameTag[] = "IMAGE";
-constexpr char kGpuBufferTag[] = "IMAGE_GPU";
-}  // namespace
-
 namespace mediapipe {
+namespace api2 {
+
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+using GpuBuffer = AnyType;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
 
 // Extracts image properties from the input image and outputs the properties.
 // Currently only supports image size.
 // Input:
 //   One of the following:
-//   IMAGE: An ImageFrame
+//   IMAGE: An Image or ImageFrame (for backward compatibility with existing
+//          graphs that use IMAGE for ImageFrame input)
+//   IMAGE_CPU: An ImageFrame
 //   IMAGE_GPU: A GpuBuffer
 //
 // Output:
@@ -42,59 +49,64 @@ namespace mediapipe {
 //   input_stream: "IMAGE:image"
 //   output_stream: "SIZE:size"
 // }
-class ImagePropertiesCalculator : public CalculatorBase {
+class ImagePropertiesCalculator : public Node {
  public:
-  static absl::Status GetContract(CalculatorContract* cc) {
+  static constexpr Input<
-    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+      OneOf<mediapipe::Image, mediapipe::ImageFrame>>::Optional kIn{"IMAGE"};
-              cc->Inputs().HasTag(kGpuBufferTag));
+  // IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level
-    if (cc->Inputs().HasTag(kImageFrameTag)) {
+  // graphs for the Python Solution APIs to figure out the type of input stream
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+  // without running into ambiguities from IMAGE.
-    }
+  // TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image.
-#if !MEDIAPIPE_DISABLE_GPU
+  static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
-    if (cc->Inputs().HasTag(kGpuBufferTag)) {
+  static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
-      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
+  static constexpr Output<std::pair<int, int>> kOut{"SIZE"};
-    }
-#endif  // !MEDIAPIPE_DISABLE_GPU
 
-    if (cc->Outputs().HasTag("SIZE")) {
+  MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
-      cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
-    }
 
-    return absl::OkStatus();
+  static absl::Status UpdateContract(CalculatorContract* cc) {
-  }
+    RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() +
+                     kInGpu(cc).IsConnected(),
+                 1)
+        << "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is "
+           "expected.";
 
-  absl::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) override {
-    int width;
+    std::pair<int, int> size;
-    int height;
 
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
+    if (kIn(cc).IsConnected()) {
-        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
+      kIn(cc).Visit(
-      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
+          [&size](const mediapipe::Image& value) {
-      width = image.Width();
+            size.first = value.width();
-      height = image.Height();
+            size.second = value.height();
+          },
+          [&size](const mediapipe::ImageFrame& value) {
+            size.first = value.Width();
+            size.second = value.Height();
+          });
+    }
+    if (kInCpu(cc).IsConnected()) {
+      const auto& image = *kInCpu(cc);
+      size.first = image.Width();
+      size.second = image.Height();
     }
 #if !MEDIAPIPE_DISABLE_GPU
-    if (cc->Inputs().HasTag(kGpuBufferTag) &&
+    if (kInGpu(cc).IsConnected()) {
-        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
+      const auto& image = *kInGpu(cc);
-      const auto& image =
+      size.first = image.width();
-          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
+      size.second = image.height();
-      width = image.width();
-      height = image.height();
     }
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-    cc->Outputs().Tag("SIZE").AddPacket(
+    kOut(cc).Send(size);
-        MakePacket<std::pair<int, int>>(width, height)
-            .At(cc->InputTimestamp()));
 
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(ImagePropertiesCalculator);
 
+MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -546,7 +546,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
   auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
                                                   input.format());
 
-  gpu_helper_.BindFramebuffer(dst);  // GL_TEXTURE0
+  gpu_helper_.BindFramebuffer(dst);
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc

@@ -36,11 +36,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
   Packet input_packet = MakePacket<std::string>(contents);
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));
@@ -79,11 +79,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
       reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));

mediapipe/calculators/image/recolor_calculator.cc

@@ -209,6 +209,9 @@ absl::Status RecolorCalculator::Close(CalculatorContext* cc) {
 
 absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
   if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
+    cc->Outputs()
+        .Tag(kImageFrameTag)
+        .AddPacket(cc->Inputs().Tag(kImageFrameTag).Value());
     return absl::OkStatus();
   }
   // Get inputs and setup output.
@@ -270,6 +273,9 @@ absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
 
 absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
   if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
+    cc->Outputs()
+        .Tag(kGpuBufferTag)
+        .AddPacket(cc->Inputs().Tag(kGpuBufferTag).Value());
     return absl::OkStatus();
   }
 #if !MEDIAPIPE_DISABLE_GPU
@@ -287,7 +293,7 @@ absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
 
   // Run recolor shader on GPU.
   {
-    gpu_helper_.BindFramebuffer(dst_tex);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(dst_tex);
 
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(img_tex.target(), img_tex.name());

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -323,7 +323,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
     const auto& alpha_mask =
         cc->Inputs().Tag(kInputAlphaTagGpu).Get<mediapipe::GpuBuffer>();
     auto alpha_texture = gpu_helper_.CreateSourceTexture(alpha_mask);
-    gpu_helper_.BindFramebuffer(output_texture);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(output_texture);
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(GL_TEXTURE_2D, input_texture.name());
     glActiveTexture(GL_TEXTURE2);
@@ -335,7 +335,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
     glBindTexture(GL_TEXTURE_2D, 0);
     alpha_texture.Release();
   } else {
-    gpu_helper_.BindFramebuffer(output_texture);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(output_texture);
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(GL_TEXTURE_2D, input_texture.name());
     GlRender(cc);  // use value from options

mediapipe/calculators/tensor/BUILD

@@ -490,6 +490,7 @@ cc_library(
         "//mediapipe/framework/port:statusor",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
@@ -526,6 +527,7 @@ mediapipe_proto_library(
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
+        "//mediapipe/gpu:gpu_origin_proto",
     ],
 )
 
@@ -561,13 +563,13 @@ cc_test(
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
@@ -583,6 +585,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_utils",
         "//mediapipe/framework/formats:image",

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -31,6 +31,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
@@ -236,7 +237,7 @@ class ImageToTensorCalculator : public Node {
   }
 
  private:
-  bool DoesInputStartAtBottom() {
+  bool DoesGpuInputStartAtBottom() {
     return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
   }
 
@@ -290,11 +291,11 @@ class ImageToTensorCalculator : public Node {
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
         ASSIGN_OR_RETURN(gpu_converter_,
                          CreateImageToGlBufferTensorConverter(
-                             cc, DoesInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #else
         ASSIGN_OR_RETURN(gpu_converter_,
                          CreateImageToGlTextureTensorConverter(
-                             cc, DoesInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #endif  // MEDIAPIPE_METAL_ENABLED
 #endif  // !MEDIAPIPE_DISABLE_GPU
       }

mediapipe/calculators/tensor/image_to_tensor_calculator.proto

@@ -17,20 +17,7 @@ syntax = "proto2";
 package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
-
+import "mediapipe/gpu/gpu_origin.proto";
-message GpuOrigin {
-  enum Mode {
-    DEFAULT = 0;
-
-    // OpenGL: bottom-left origin
-    // Metal : top-left origin
-    CONVENTIONAL = 1;
-
-    // OpenGL: top-left origin
-    // Metal : top-left origin
-    TOP_LEFT = 2;
-  }
-}
 
 message ImageToTensorCalculatorOptions {
   extend mediapipe.CalculatorOptions {

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -15,6 +15,7 @@
 #include <cmath>
 #include <vector>
 
+#include "absl/flags/flag.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
@@ -28,7 +29,6 @@
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter {
       return absl::OkStatus();
     }));
 
-    return tensor;
+    return std::move(tensor);
   }
 
   ~GlProcessor() override {

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
           tflite::gpu::HW(output_dims.height, output_dims.width),
           command_buffer, buffer_view.buffer()));
       [command_buffer commit];
-      return tensor;
+      return std::move(tensor);
     }
   }
 

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc

@@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
         GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                     range_min, range_max));
     transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
-    return tensor;
+    return std::move(tensor);
   }
 
  private:

mediapipe/calculators/tensor/inference_calculator.cc

@@ -41,7 +41,6 @@ class InferenceCalculatorSelectorImpl
         (options.has_delegate() && options.delegate().has_gpu());
     if (should_use_gpu) {
       impls.emplace_back("Metal");
-      impls.emplace_back("MlDrift");
       impls.emplace_back("Gl");
     }
     impls.emplace_back("Cpu");

mediapipe/calculators/tensor/inference_calculator.h

@@ -118,10 +118,6 @@ struct InferenceCalculatorGl : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
 };
 
-struct InferenceCalculatorMlDrift : public InferenceCalculator {
-  static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
-};
-
 struct InferenceCalculatorMetal : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
 };

mediapipe/calculators/tensor/inference_calculator.proto

@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Choose any of available APIs to force running inference using it.
-      enum API {
+      enum Api {
         ANY = 0;
         OPENGL = 1;
         OPENCL = 2;
       }
-      optional API api = 4 [default = ANY];
+      optional Api api = 4 [default = ANY];
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Set to true to use 16-bit float precision. If max precision is needed,

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -136,7 +136,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);

mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc

@@ -57,11 +57,7 @@ const std::vector<Param>& GetParams() {
     // Metal is not available on the iOS simulator.
     p.push_back({"Metal", "Metal"});
     p.back().delegate.mutable_gpu();
-#endif  // TARGET_IPHONE_SIMULATOR
+#endif                // TARGET_IPHONE_SIMULATOR
-#if __EMSCRIPTEN__
-    p.push_back({"MlDrift", "MlDrift"});
-    p.back().delegate.mutable_gpu();
-#endif                // __EMSCRIPTEN__
 #if __ANDROID__ && 0  // Disabled for now since emulator can't go GLESv3
     p.push_back({"Gl", "Gl"});
     p.back().delegate.mutable_gpu();
@@ -78,18 +74,6 @@ const std::vector<Param>& GetParams() {
 
 class InferenceCalculatorTest : public testing::TestWithParam<Param> {
  protected:
-#if __EMSCRIPTEN__
-  // TODO: fix Tensor locking.
-  // The MlDrift backend currently fails in debug mode without this,
-  // because of Tensor locking issues. I am adding this temporarily since
-  // the calculator is already being used and it's better to have test
-  // coverage for it. Also, the issue doesn't apply to our Emscripten
-  // build in practice since it's single-threaded.
-  void SetUp(void) override {
-    absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kIgnore);
-  }
-#endif  // __EMSCRIPTEN__
-
   void SetDelegateForParam(mediapipe::CalculatorGraphConfig_Node* node) {
     *node->mutable_options()
          ->MutableExtension(mediapipe::InferenceCalculatorOptions::ext)

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -63,7 +63,7 @@ class InferenceCalculatorGlImpl
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
   bool allow_precision_loss_ = false;
-  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::API
+  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
       tflite_gpu_runner_api_;
 #endif  // MEDIAPIPE_TFLITE_GL_INFERENCE
 
@@ -244,7 +244,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   // Create runner
   tflite::gpu::InferenceOptions options;
@@ -294,7 +294,7 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);
@@ -317,7 +317,8 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
 absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
   // Configure and create the delegate.
   TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
-  options.compile_options.precision_loss_allowed = 1;
+  options.compile_options.precision_loss_allowed =
+      allow_precision_loss_ ? 1 : 0;
   options.compile_options.preferred_gl_object_type =
       TFLITE_GL_OBJECT_TYPE_FASTEST;
   options.compile_options.dynamic_batch_enabled = 0;

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -97,6 +97,7 @@ class InferenceCalculatorMetalImpl
   Packet<TfLiteModelPtr> model_packet_;
   std::unique_ptr<tflite::Interpreter> interpreter_;
   TfLiteDelegatePtr delegate_;
+  bool allow_precision_loss_ = false;
 
 #if MEDIAPIPE_TFLITE_METAL_INFERENCE
   MPPMetalHelper* gpu_helper_ = nullptr;
@@ -122,6 +123,9 @@ absl::Status InferenceCalculatorMetalImpl::UpdateContract(
 }
 
 absl::Status InferenceCalculatorMetalImpl::Open(CalculatorContext* cc) {
+  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
+  allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
+
   MP_RETURN_IF_ERROR(LoadModel(cc));
 
   gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
@@ -200,7 +204,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);
@@ -222,7 +226,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
 
   // Configure and create the delegate.
   TFLGpuDelegateOptions options;
-  options.allow_precision_loss = true;
+  options.allow_precision_loss = allow_precision_loss_;
   options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypeDoNotWait;
   delegate_ =
       TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), &TFLGpuDelegateDelete);
@@ -239,7 +243,9 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
                           tensor->dims->data + tensor->dims->size};
     dims.back() = RoundUp(dims.back(), 4);
     gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
-        Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
+        allow_precision_loss_ ? Tensor::ElementType::kFloat16
+                              : Tensor::ElementType::kFloat32,
+        Tensor::Shape{dims}));
     auto buffer_view =
         gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice);
     RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
@@ -261,7 +267,9 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
     output_shapes_[i] = {dims};
     dims.back() = RoundUp(dims.back(), 4);
     gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
-        Tensor::ElementType::kFloat16, Tensor::Shape{dims}));
+        allow_precision_loss_ ? Tensor::ElementType::kFloat16
+                              : Tensor::ElementType::kFloat32,
+        Tensor::Shape{dims}));
     RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                      delegate_.get(), output_indices[i],
                      gpu_buffers_out_[i]
@@ -271,17 +279,19 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
   }
 
   // Create converter for GPU input.
-  converter_to_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
+  converter_to_BPHWC4_ =
-                                                        isFloat16:true
+      [[TFLBufferConvert alloc] initWithDevice:device
-                                                  convertToPBHWC4:true];
+                                     isFloat16:allow_precision_loss_
+                               convertToPBHWC4:true];
   if (converter_to_BPHWC4_ == nil) {
     return mediapipe::InternalError(
         "Error initializating input buffer converter");
   }
   // Create converter for GPU output.
-  converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
+  converter_from_BPHWC4_ =
-                                                          isFloat16:true
+      [[TFLBufferConvert alloc] initWithDevice:device
-                                                    convertToPBHWC4:false];
+                                     isFloat16:allow_precision_loss_
+                               convertToPBHWC4:false];
   if (converter_from_BPHWC4_ == nil) {
     return absl::InternalError("Error initializating output buffer converter");
   }

mediapipe/calculators/tensor/tensor_converter_calculator_test.cc

@@ -84,7 +84,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TensorConverterCalculator"
@@ -96,7 +96,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -146,7 +146,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TensorConverterCalculator"
@@ -158,7 +158,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -205,7 +205,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
   CalculatorGraph graph;
   // Run the calculator and verify that one output is generated.
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_image"
         node {
           calculator: "TensorConverterCalculator"
@@ -220,7 +220,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
             }
           }
         }
-      )");
+      )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor", &graph_config, &output_packets);
 

mediapipe/calculators/tensor/tensors_to_classification_calculator.cc

@@ -89,7 +89,8 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
     ASSIGN_OR_RETURN(string_path,
                      PathToResourceAsFile(options_.label_map_path()));
     std::string label_map_string;
-    MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+    MP_RETURN_IF_ERROR(
+        mediapipe::GetResourceContents(string_path, &label_map_string));
 
     std::istringstream stream(label_map_string);
     std::string line;
@@ -98,6 +99,14 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
       label_map_[i++] = line;
     }
     label_map_loaded_ = true;
+  } else if (options_.has_label_map()) {
+    for (int i = 0; i < options_.label_map().entries_size(); ++i) {
+      const auto& entry = options_.label_map().entries(i);
+      RET_CHECK(!label_map_.contains(entry.id()))
+          << "Duplicate id found: " << entry.id();
+      label_map_[entry.id()] = entry.label();
+    }
+    label_map_loaded_ = true;
   }
 
   return absl::OkStatus();

mediapipe/calculators/tensor/tensors_to_classification_calculator.proto

@@ -25,6 +25,14 @@ message TensorsToClassificationCalculatorOptions {
     optional TensorsToClassificationCalculatorOptions ext = 335742638;
   }
 
+  message LabelMap {
+    message Entry {
+      optional int32 id = 1;
+      optional string label = 2;
+    }
+    repeated Entry entries = 1;
+  }
+
   // Score threshold for perserving the class.
   optional float min_score_threshold = 1;
   // Number of highest scoring labels to output.  If top_k is not positive then
@@ -32,6 +40,10 @@ message TensorsToClassificationCalculatorOptions {
   optional int32 top_k = 2;
   // Path to a label map file for getting the actual name of class ids.
   optional string label_map_path = 3;
+  // Label map. (Can be used instead of label_map_path.)
+  // NOTE: "label_map_path", if specified, takes precedence over "label_map".
+  optional LabelMap label_map = 5;
+
   // Whether the input is a single float for binary classification.
   // When true, only a single float is expected in the input tensor and the
   // label map, if provided, is expected to have exactly two labels.

mediapipe/calculators/tensor/tensors_to_classification_calculator_test.cc

@@ -56,14 +56,14 @@ class TensorsToClassificationCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
     options {
       [mediapipe.TensorsToClassificationCalculatorOptions.ext] {}
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -85,7 +85,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
 }
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -94,7 +94,42 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
         label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt"
       }
     }
-  )"));
+  )pb"));
+
+  BuildGraph(&runner, {0, 0.5, 1});
+  MP_ASSERT_OK(runner.Run());
+
+  const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
+
+  EXPECT_EQ(1, output_packets_.size());
+
+  const auto& classification_list =
+      output_packets_[0].Get<ClassificationList>();
+  EXPECT_EQ(3, classification_list.classification_size());
+
+  // Verify that the label field is set.
+  for (int i = 0; i < classification_list.classification_size(); ++i) {
+    EXPECT_EQ(i, classification_list.classification(i).index());
+    EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
+    ASSERT_TRUE(classification_list.classification(i).has_label());
+  }
+}
+
+TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMap) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
+    calculator: "TensorsToClassificationCalculator"
+    input_stream: "TENSORS:tensors"
+    output_stream: "CLASSIFICATIONS:classifications"
+    options {
+      [mediapipe.TensorsToClassificationCalculatorOptions.ext] {
+        label_map {
+          entries { id: 0, label: "ClassA" }
+          entries { id: 1, label: "ClassB" }
+          entries { id: 2, label: "ClassC" }
+        }
+      }
+    }
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -117,7 +152,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
 
 TEST_F(TensorsToClassificationCalculatorTest,
        CorrectOutputWithLabelMinScoreThreshold) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -126,7 +161,7 @@ TEST_F(TensorsToClassificationCalculatorTest,
         min_score_threshold: 0.6
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -144,14 +179,14 @@ TEST_F(TensorsToClassificationCalculatorTest,
 }
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
     options {
       [mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());

mediapipe/calculators/tensor/tensors_to_floats_calculator_test.cc

@@ -57,11 +57,11 @@ class TensorsToFloatsCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOAT:float"
-  )"));
+  )pb"));
 
   const float single_value = 0.5;
   BuildGraph(&runner, {single_value});
@@ -76,11 +76,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
-  )"));
+  )pb"));
 
   const float single_value = 0.5;
   BuildGraph(&runner, {single_value});
@@ -95,11 +95,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
-  )"));
+  )pb"));
 
   const std::vector<float> input_values = {0.f, 0.5f, 1.0f};
   BuildGraph(&runner, input_values);
@@ -116,14 +116,14 @@ TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, FloatVectorWithSigmoid) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
     options {
       [mediapipe.TensorsToFloatsCalculatorOptions.ext] { activation: SIGMOID }
     }
-  )"));
+  )pb"));
 
   const std::vector<float> input_values = {-1.f, 0.f, 1.0f};
   const std::vector<float> expected_output_with_sigmoid = {0.269f, 0.5f,

mediapipe/calculators/tensorflow/BUILD

@@ -892,13 +892,13 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
@@ -923,13 +923,13 @@ cc_test(
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
@@ -954,11 +954,11 @@ cc_test(
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
@@ -981,11 +981,11 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
@@ -1144,8 +1144,8 @@ cc_test(
         ":tensorflow_inference_calculator",
         ":tensorflow_session_from_frozen_graph_generator",
         ":tensorflow_session_from_frozen_graph_generator_cc_proto",
+        "@com_google_absl//absl/flags:flag",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",

mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc

@@ -68,31 +68,31 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
 
   void CreateNodeConfig(CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "NUM_DETECTIONS:num_detections"
       input_stream: "BOXES:boxes"
       input_stream: "SCORES:scores"
       input_stream: "CLASSES:classes"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void CreateNodeConfigRawTensors(
       CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "BOXES:raw_detection_boxes"
       input_stream: "SCORES:raw_detection_scores"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void CreateNodeConfigWithKeypoints(
       CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "NUM_DETECTIONS:num_detections"
       input_stream: "BOXES:boxes"
@@ -100,7 +100,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
       input_stream: "CLASSES:classes"
       input_stream: "KEYPOINTS:keypoints"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void SetUpCalculatorRunner() {
@@ -177,7 +177,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
     InsertExtraSingltonDim(&input_scores_);
     InsertExtraSingltonDim(&input_classes_);
     CalculatorGraphConfig::Node node_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
           calculator: "ObjectDetectionTensorsToDetectionsCalculator"
           input_stream: "NUM_DETECTIONS:num_detections"
           input_stream: "BOXES:boxes"
@@ -188,7 +188,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
             [mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions
                  .ext]: { tensor_dim_to_squeeze: 0 }
           }
-        )");
+        )pb");
     runner_ = absl::make_unique<CalculatorRunner>(node_config);
     runner_->MutableInputs()
         ->Tag(kNumDetections)

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -267,6 +267,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
     }
   }
 
+  absl::Status VerifySize() {
+    const int64 MAX_PROTO_BYTES = 1073741823;
+    std::string id = mpms::HasExampleId(*sequence_)
+                         ? mpms::GetExampleId(*sequence_)
+                         : "example";
+    RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES)
+        << "sequence '" << id
+        << "' would be too many bytes to serialize after adding features.";
+    return absl::OkStatus();
+  }
+
   absl::Status Close(CalculatorContext* cc) override {
     auto& options = cc->Options<PackMediaSequenceCalculatorOptions>();
     if (options.reconcile_metadata()) {
@@ -275,6 +286,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           options.reconcile_region_annotations(), sequence_.get()));
     }
 
+    if (options.skip_large_sequences()) {
+      RET_CHECK_OK(VerifySize());
+    }
     if (options.output_only_if_all_present()) {
       absl::Status status = VerifySequence();
       if (!status.ok()) {

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto

@@ -61,4 +61,8 @@ message PackMediaSequenceCalculatorOptions {
   // present, the previous images and timestamps will be removed before adding
   // the new images.
   optional bool replace_data_instead_of_append = 4 [default = true];
+
+  // If true, will return an error status if an output sequence would be too
+  // many bytes to serialize.
+  optional bool skip_large_sequences = 7 [default = true];
 }

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -889,5 +889,24 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
   MP_ASSERT_OK(runner_->Run());
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
+  SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+
+  // 1 billion floats should be > 1GB which can't be serialized. It should fail
+  // gracefully with this input.
+  int num_timesteps = 1000;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
+    runner_->MutableInputs()
+        ->Tag("FLOAT_FEATURE_TEST")
+        .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
+  }
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+  ASSERT_FALSE(runner_->Run().ok());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc

@@ -34,15 +34,28 @@ constexpr char kTensor[] = "TENSOR";
 }  // namespace
 
 // Input:
-//  Tensor of type DT_FLOAT, with values between 0-255 (SRGB or GRAY8). The
+//  Tensor of type DT_FLOAT or DT_UINT8, with values between 0-255
-//  shape can be HxWx{3,1} or simply HxW.
+//  (SRGB or GRAY8). The shape can be HxWx{3,1} or simply HxW.
 //
-//  Optionally supports a scale factor that can scale 0-1 value ranges to 0-255.
+//  For DT_FLOAT tensors, optionally supports a scale factor that can scale 0-1
+//  value ranges to 0-255.
 //
 // Output:
 //  ImageFrame containing the values of the tensor cast as uint8 (SRGB or GRAY8)
 //
 // Possible extensions: support other input ranges, maybe 4D tensors.
+//
+// Example:
+// node {
+//   calculator: "TensorToImageFrameCalculator"
+//   input_stream: "TENSOR:3d_float_tensor"
+//   output_stream: "IMAGE:image_frame"
+//   options {
+//     [mediapipe.TensorToImageFrameCalculatorOptions.ext] {
+//       scale_factor: 1.0  # set to 255.0 for [0,1] -> [0,255] scaling
+//     }
+//   }
+// }
 class TensorToImageFrameCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc);
@@ -57,8 +70,8 @@ class TensorToImageFrameCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(TensorToImageFrameCalculator);
 
 absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
-  RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
-      << "Only one input stream is supported.";
+      << "Only one output stream is supported.";
   RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
       << "One input stream must be provided.";
   RET_CHECK(cc->Inputs().HasTag(kTensor))
@@ -91,29 +104,44 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
       RET_CHECK_EQ(depth, 3) << "Output tensor depth must be 3 or 1.";
     }
   }
-  const int32 total_size =
+  int32 height = input_tensor.dim_size(0);
-      input_tensor.dim_size(0) * input_tensor.dim_size(1) * depth;
+  int32 width = input_tensor.dim_size(1);
-  std::unique_ptr<uint8[]> buffer(new uint8[total_size]);
+  auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
-  auto data = input_tensor.flat<float>().data();
+  const int32 total_size = height * width * depth;
-  for (int i = 0; i < total_size; ++i) {
+
-    float d = scale_factor_ * data[i];
+  ::std::unique_ptr<const ImageFrame> output;
-    if (d < 0) d = 0;
+  if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
-    if (d > 255) d = 255;
+    // Allocate buffer with alignments.
-    buffer[i] = d;
+    std::unique_ptr<uint8_t[]> buffer(
+        new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
+    auto data = input_tensor.flat<float>().data();
+    for (int i = 0; i < total_size; ++i) {
+      float d = scale_factor_ * data[i];
+      if (d < 0) d = 0;
+      if (d > 255) d = 255;
+      buffer[i] = d;
+    }
+    output = ::absl::make_unique<ImageFrame>(format, width, height,
+                                             width * depth, buffer.release());
+  } else if (input_tensor.dtype() == tensorflow::DT_UINT8) {
+    if (scale_factor_ != 1.0) {
+      return absl::InvalidArgumentError("scale_factor_ given for uint8 tensor");
+    }
+    // tf::Tensor has internally ref-counted buffer. The following code make the
+    // ImageFrame own the copied Tensor through the deleter, which increases
+    // the refcount of the buffer and allow us to use the shared buffer as the
+    // image. This allows us to create an ImageFrame object without copying
+    // buffer. const ImageFrame prevents the buffer from being modified later.
+    auto copy = new tf::Tensor(input_tensor);
+    output = ::absl::make_unique<const ImageFrame>(
+        format, width, height, width * depth, copy->flat<uint8_t>().data(),
+        [copy](uint8*) { delete copy; });
+  } else {
+    return absl::InvalidArgumentError(
+        absl::StrCat("Expected float or uint8 tensor, received ",
+                     DataTypeString(input_tensor.dtype())));
   }
 
-  ::std::unique_ptr<ImageFrame> output;
-  if (depth == 3) {
-    output = ::absl::make_unique<ImageFrame>(
-        ImageFormat::SRGB, input_tensor.dim_size(1), input_tensor.dim_size(0),
-        input_tensor.dim_size(1) * 3, buffer.release());
-  } else if (depth == 1) {
-    output = ::absl::make_unique<ImageFrame>(
-        ImageFormat::GRAY8, input_tensor.dim_size(1), input_tensor.dim_size(0),
-        input_tensor.dim_size(1), buffer.release());
-  } else {
-    return absl::InvalidArgumentError("Unrecognized image depth.");
-  }
   cc->Outputs().Tag(kImage).Add(output.release(), cc->InputTimestamp());
 
   return absl::OkStatus();

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc

@@ -29,6 +29,7 @@ constexpr char kImage[] = "IMAGE";
 
 }  // namespace
 
+template <class TypeParam>
 class TensorToImageFrameCalculatorTest : public ::testing::Test {
  protected:
   void SetUpRunner() {
@@ -42,14 +43,20 @@ class TensorToImageFrameCalculatorTest : public ::testing::Test {
   std::unique_ptr<CalculatorRunner> runner_;
 };
 
-TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
+using TensorToImageFrameCalculatorTestTypes = ::testing::Types<float, uint8_t>;
-  SetUpRunner();
+TYPED_TEST_CASE(TensorToImageFrameCalculatorTest,
+                TensorToImageFrameCalculatorTestTypes);
+
+TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
+  // TYPED_TEST requires explicit "this->"
+  this->SetUpRunner();
+  auto& runner = this->runner_;
   constexpr int kWidth = 16;
   constexpr int kHeight = 8;
-  const tf::TensorShape tensor_shape(
+  const tf::TensorShape tensor_shape{kHeight, kWidth, 3};
-      std::vector<tf::int64>{kHeight, kWidth, 3});
+  auto tensor = absl::make_unique<tf::Tensor>(
-  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
-  auto tensor_vec = tensor->flat<float>().data();
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
 
   // Writing sequence of integers as floats which we want back (as they were
   // written).
@@ -58,15 +65,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
   }
 
   const int64 time = 1234;
-  runner_->MutableInputs()->Tag(kTensor).packets.push_back(
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
       Adopt(tensor.release()).At(Timestamp(time)));
 
-  EXPECT_TRUE(runner_->Run().ok());
+  EXPECT_TRUE(runner->Run().ok());
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag(kImage).packets;
+      runner->Outputs().Tag(kImage).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+  EXPECT_EQ(ImageFormat::SRGB, output_image.Format());
   EXPECT_EQ(kWidth, output_image.Width());
   EXPECT_EQ(kHeight, output_image.Height());
 
@@ -76,14 +84,15 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) {
   }
 }
 
-TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
+TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
-  SetUpRunner();
+  this->SetUpRunner();
+  auto& runner = this->runner_;
   constexpr int kWidth = 16;
   constexpr int kHeight = 8;
-  const tf::TensorShape tensor_shape(
+  const tf::TensorShape tensor_shape{kHeight, kWidth, 1};
-      std::vector<tf::int64>{kHeight, kWidth, 1});
+  auto tensor = absl::make_unique<tf::Tensor>(
-  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
-  auto tensor_vec = tensor->flat<float>().data();
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
 
   // Writing sequence of integers as floats which we want back (as they were
   // written).
@@ -92,15 +101,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
   }
 
   const int64 time = 1234;
-  runner_->MutableInputs()->Tag(kTensor).packets.push_back(
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
       Adopt(tensor.release()).At(Timestamp(time)));
 
-  EXPECT_TRUE(runner_->Run().ok());
+  EXPECT_TRUE(runner->Run().ok());
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag(kImage).packets;
+      runner->Outputs().Tag(kImage).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+  EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
   EXPECT_EQ(kWidth, output_image.Width());
   EXPECT_EQ(kHeight, output_image.Height());
 
@@ -110,13 +120,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) {
   }
 }
 
-TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame2DGray) {
+TYPED_TEST(TensorToImageFrameCalculatorTest,
-  SetUpRunner();
+           Converts3DTensorToImageFrame2DGray) {
+  this->SetUpRunner();
+  auto& runner = this->runner_;
   constexpr int kWidth = 16;
   constexpr int kHeight = 8;
-  const tf::TensorShape tensor_shape(std::vector<tf::int64>{kHeight, kWidth});
+  const tf::TensorShape tensor_shape{kHeight, kWidth};
-  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_FLOAT, tensor_shape);
+  auto tensor = absl::make_unique<tf::Tensor>(
-  auto tensor_vec = tensor->flat<float>().data();
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
 
   // Writing sequence of integers as floats which we want back (as they were
   // written).
@@ -125,15 +138,16 @@ TEST_F(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame2DGray) {
   }
 
   const int64 time = 1234;
-  runner_->MutableInputs()->Tag(kTensor).packets.push_back(
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
       Adopt(tensor.release()).At(Timestamp(time)));
 
-  EXPECT_TRUE(runner_->Run().ok());
+  EXPECT_TRUE(runner->Run().ok());
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag(kImage).packets;
+      runner->Outputs().Tag(kImage).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+  EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
   EXPECT_EQ(kWidth, output_image.Width());
   EXPECT_EQ(kHeight, output_image.Height());
 

mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc

@@ -91,8 +91,6 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
 // the input data when it arrives in Process(). In particular, if the header
 // states that we produce a 1xD column vector, the input tensor must also be 1xD
 //
-// This designed was discussed in http://g/speakeranalysis/4uyx7cNRwJY and
-// http://g/daredevil-project/VB26tcseUy8.
 // Example Config
 // node: {
 //   calculator: "TensorToMatrixCalculator"
@@ -158,22 +156,17 @@ absl::Status TensorToMatrixCalculator::Open(CalculatorContext* cc) {
   if (header_status.ok()) {
     if (cc->Options<TensorToMatrixCalculatorOptions>()
             .has_time_series_header_overrides()) {
-      // From design discussions with Daredevil, we only want to support single
+      // This only supports a single sample per packet for now, so we hardcode
-      // sample per packet for now, so we hardcode the sample_rate based on the
+      // the sample_rate based on the packet_rate of the REFERENCE and fail
-      // packet_rate of the REFERENCE and fail noisily if we cannot. An
+      // if we cannot.
-      // alternative would be to calculate the sample_rate from the reference
-      // sample_rate and the change in num_samples between the reference and
-      // override headers:
-      // sample_rate_output = sample_rate_reference /
-      //                      (num_samples_override / num_samples_reference)
       const TimeSeriesHeader& override_header =
           cc->Options<TensorToMatrixCalculatorOptions>()
               .time_series_header_overrides();
       input_header->MergeFrom(override_header);
-      CHECK(input_header->has_packet_rate())
+      RET_CHECK(input_header->has_packet_rate())
           << "The TimeSeriesHeader.packet_rate must be set.";
       if (!override_header.has_sample_rate()) {
-        CHECK_EQ(input_header->num_samples(), 1)
+        RET_CHECK_EQ(input_header->num_samples(), 1)
             << "Currently the time series can only output single samples.";
         input_header->set_sample_rate(input_header->packet_rate());
       }
@@ -186,20 +179,16 @@ absl::Status TensorToMatrixCalculator::Open(CalculatorContext* cc) {
 }
 
 absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) {
-  // Daredevil requested CHECK for noisy failures rather than quieter RET_CHECK
-  // failures. These are absolute conditions of the graph for the graph to be
-  // valid, and if it is violated by any input anywhere, the graph will be
-  // invalid for all inputs. A hard CHECK will enable faster debugging by
-  // immediately exiting and more prominently displaying error messages.
-  // Do not replace with RET_CHECKs.
-
   // Verify that each reference stream packet corresponds to a tensor packet
   // otherwise the header information is invalid. If we don't have a reference
   // stream, Process() is only called when we have an input tensor and this is
   // always True.
-  CHECK(cc->Inputs().HasTag(kTensor))
+  RET_CHECK(cc->Inputs().HasTag(kTensor))
       << "Tensor stream not available at same timestamp as the reference "
          "stream.";
+  RET_CHECK(!cc->Inputs().Tag(kTensor).IsEmpty()) << "Tensor stream is empty.";
+  RET_CHECK_OK(cc->Inputs().Tag(kTensor).Value().ValidateAsType<tf::Tensor>())
+      << "Tensor stream packet does not contain a Tensor.";
 
   const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>();
   CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims())
@@ -207,13 +196,12 @@ absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) {
   const int32 length = input_tensor.dim_size(input_tensor.dims() - 1);
   const int32 width = (1 == input_tensor.dims()) ? 1 : input_tensor.dim_size(0);
   if (header_.has_num_channels()) {
-    CHECK_EQ(length, header_.num_channels())
+    RET_CHECK_EQ(length, header_.num_channels())
         << "The number of channels at runtime does not match the header.";
   }
   if (header_.has_num_samples()) {
-    CHECK_EQ(width, header_.num_samples())
+    RET_CHECK_EQ(width, header_.num_samples())
         << "The number of samples at runtime does not match the header.";
-    ;
   }
   auto output = absl::make_unique<Matrix>(width, length);
   *output =

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -98,388 +98,543 @@ class InferenceState {
 
 // This calculator performs inference on a trained TensorFlow model.
 //
-// A mediapipe::TensorFlowSession with a model loaded and ready for use.
+// TensorFlow Sessions can be created from checkpoint paths, frozen models, or
-// For this calculator it must include a tag_to_tensor_map.
+// the SavedModel system. See the TensorFlowSessionFrom* packet generators for
-cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
+// details. Each of these methods defines a mapping between MediaPipe streams
-if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
+// and TensorFlow tensors. All of this information is passed in as an
-  cc->InputSidePackets()
+// input_side_packet.
-      .Tag("RECURRENT_INIT_TENSORS")
+//
-      .Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
+// The input and output streams are TensorFlow tensors labeled by tags. The tags
-}
+// for the streams are matched to feeds and fetchs in a TensorFlow session using
-return absl::OkStatus();
+// a named_signature.generic_signature in the ModelManifest. The
-}
+// generic_signature is used as key-value pairs between the MediaPipe tag and
+// the TensorFlow tensor. The signature_name in the options proto determines
+// which named_signature is used. The keys in the generic_signature must be
+// valid MediaPipe tags ([A-Z0-9_]*, no lowercase or special characters). All of
+// the tensors corresponding to tags in the signature for input_streams are fed
+// to the model and for output_streams the tensors are fetched from the model.
+//
+// Other calculators are used to convert data to and from tensors, this op only
+// handles the TensorFlow session and batching. Batching occurs by concatenating
+// input tensors along the 0th dimension across timestamps. If the 0th dimension
+// is not a batch dimension, this calculator will add a 0th dimension by
+// default. Setting add_batch_dim_to_tensors to false disables the dimension
+// addition. Once batch_size inputs have been provided, the batch will be run
+// and the output tensors sent out on the output streams with timestamps
+// corresponding to the input stream packets. Setting the batch_size to 1
+// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
+//
+// The TensorFlowInferenceCalculator also support feeding states recurrently for
+// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the
+// recurrent tensors. Initializing the recurrent state can be handled by the
+// GraphTensorsPacketGenerator.
+//
+// The calculator updates two Counters to report timing information:
+//   --<name>-TotalTimeUsecs = Total time spent running inference (in usecs),
+//   --<name>-TotalProcessedTimestamps = # of instances processed
+//         (approximately batches processed  * batch_size),
+// where <name> is replaced with CalculatorGraphConfig::Node::name() if it
+// exists, or with TensorFlowInferenceCalculator if the name is not set. The
+// name must be set for timing information to be instance-specific in graphs
+// with multiple TensorFlowInferenceCalculators.
+//
+// Example config:
+//   packet_generator {
+//     packet_generator: "TensorFlowSessionFromSavedModelGenerator"
+//     output_side_packet: "tensorflow_session"
+//     options {
+//       [mediapipe.TensorFlowSessionFromSavedModelGeneratorOptions.ext]: {
+//         saved_model_path: "/path/to/saved/model"
+//         signature_name: "mediapipe"
+//       }
+//     }
+//   }
+//   node {
+//     calculator: "TensorFlowInferenceCalculator"
+//     input_stream: "IMAGES:image_tensors_keyed_in_signature_by_tag"
+//     input_stream: "AUDIO:audio_tensors_keyed_in_signature_by_tag"
+//     output_stream: "LABELS:softmax_tensor_keyed_in_signature_by_tag"
+//     input_side_packet: "SESSION:tensorflow_session"
+//   }
+//
+//   Where the input and output streams are treated as Packet<tf::Tensor> and
+//   the mediapipe_signature has tensor bindings between "IMAGES", "AUDIO", and
+//   "LABELS" and their respective tensors exported to /path/to/bundle. For an
+//   example of how this model was exported, see
+//   tensorflow_inference_test_graph_generator.py
+//
+// It is possible to use a GraphDef proto that was not exported by exporter (i.e
+// without MetaGraph with bindings). Such GraphDef could contain all of its
+// parameters in-lined (for example, it can be the output of freeze_graph.py).
+// To instantiate a TensorFlow model from a GraphDef file, replace the
+// packet_factory above with TensorFlowSessionFromFrozenGraphGenerator:
+//
+//   packet_generator {
+//     packet_generator: "TensorFlowSessionFromFrozenGraphGenerator"
+//     output_side_packet: "SESSION:tensorflow_session"
+//     options {
+//       [mediapipe.TensorFlowSessionFromFrozenGraphGeneratorOptions.ext]: {
+//         graph_proto_path: "[PATH]"
+//         tag_to_tensor_names {
+//           key: "JPG_STRING"
+//           value: "input:0"
+//         }
+//         tag_to_tensor_names {
+//           key: "SOFTMAX"
+//           value: "softmax:0"
+//         }
+//       }
+//     }
+//   }
+//
+// It is also possible to use a GraphDef proto and checkpoint file that have not
+// been frozen. This can be used to load graphs directly as they have been
+// written from training. However, it is more brittle and you are encouraged to
+// use a one of the more perminent formats described above. To instantiate a
+// TensorFlow model from a GraphDef file and checkpoint, replace the
+// packet_factory above with TensorFlowSessionFromModelCheckpointGenerator:
+//
+//   packet_generator {
+//     packet_generator: "TensorFlowSessionFromModelCheckpointGenerator"
+//     output_side_packet: "SESSION:tensorflow_session"
+//     options {
+//       [mediapipe.TensorFlowSessionFromModelCheckpointGeneratorOptions.ext]: {
+//         graph_proto_path: "[PATH]"
+//         model_options {
+//           checkpoint_path: "[PATH2]"
+//         }
+//         tag_to_tensor_names {
+//           key: "JPG_STRING"
+//           value: "input:0"
+//         }
+//         tag_to_tensor_names {
+//           key: "SOFTMAX"
+//           value: "softmax:0"
+//         }
+//       }
+//     }
+//   }
+class TensorFlowInferenceCalculator : public CalculatorBase {
+ public:
+  // Counters for recording timing information. The actual names have the value
+  // of CalculatorGraphConfig::Node::name() prepended.
+  static constexpr char kTotalUsecsCounterSuffix[] = "TotalTimeUsecs";
+  static constexpr char kTotalProcessedTimestampsCounterSuffix[] =
+      "TotalProcessedTimestamps";
+  static constexpr char kTotalSessionRunsTimeUsecsCounterSuffix[] =
+      "TotalSessionRunsTimeUsecs";
+  static constexpr char kTotalNumSessionRunsCounterSuffix[] =
+      "TotalNumSessionRuns";
 
-std::unique_ptr<InferenceState> CreateInferenceState(CalculatorContext* cc)
+  TensorFlowInferenceCalculator() : session_(nullptr) {
-    ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+    clock_ = std::unique_ptr<mediapipe::Clock>(
-  std::unique_ptr<InferenceState> inference_state =
+        mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock());
-      absl::make_unique<InferenceState>();
+  }
-  if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
+
-      !cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
+  static absl::Status GetContract(CalculatorContract* cc) {
-    std::map<std::string, tf::Tensor>* init_tensor_map;
+    const auto& options = cc->Options<TensorFlowInferenceCalculatorOptions>();
-    init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
+    RET_CHECK(!cc->Inputs().GetTags().empty());
-        cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
+    for (const std::string& tag : cc->Inputs().GetTags()) {
-    for (const auto& p : *init_tensor_map) {
+      // The tensorflow::Tensor with the tag equal to the graph node. May
-      inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
+      // have a TimeSeriesHeader if all present TimeSeriesHeaders match.
+      if (!options.batched_input()) {
+        cc->Inputs().Tag(tag).Set<tf::Tensor>();
+      } else {
+        cc->Inputs().Tag(tag).Set<std::vector<mediapipe::Packet>>();
+      }
     }
-  }
+    RET_CHECK(!cc->Outputs().GetTags().empty());
-  return inference_state;
+    for (const std::string& tag : cc->Outputs().GetTags()) {
-}
+      // The tensorflow::Tensor with tag equal to the graph node to
-
+      // output.  Any TimeSeriesHeader from the inputs will be forwarded
-absl::Status Open(CalculatorContext* cc) override {
+      // with channels set to 0.
-  options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
+      cc->Outputs().Tag(tag).Set<tf::Tensor>();
-
-  RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
-  session_ = cc->InputSidePackets()
-                 .Tag("SESSION")
-                 .Get<TensorFlowSession>()
-                 .session.get();
-  tag_to_tensor_map_ = cc->InputSidePackets()
-                           .Tag("SESSION")
-                           .Get<TensorFlowSession>()
-                           .tag_to_tensor_map;
-
-  // Validate and store the recurrent tags
-  RET_CHECK(options_.has_batch_size());
-  RET_CHECK(options_.batch_size() == 1 || options_.recurrent_tag_pair().empty())
-      << "To use recurrent_tag_pairs, batch_size must be 1.";
-  for (const auto& tag_pair : options_.recurrent_tag_pair()) {
-    const std::vector<std::string> tags = absl::StrSplit(tag_pair, ':');
-    RET_CHECK_EQ(tags.size(), 2)
-        << "recurrent_tag_pair must be a colon "
-           "separated std::string with two components: "
-        << tag_pair;
-    RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[0]))
-        << "Can't find tag '" << tags[0] << "' in signature "
-        << options_.signature_name();
-    RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[1]))
-        << "Can't find tag '" << tags[1] << "' in signature "
-        << options_.signature_name();
-    recurrent_feed_tags_.insert(tags[0]);
-    recurrent_fetch_tags_to_feed_tags_[tags[1]] = tags[0];
-  }
-
-  // Check that all tags are present in this signature bound to tensors.
-  for (const std::string& tag : cc->Inputs().GetTags()) {
-    RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
-        << "Can't find tag '" << tag << "' in signature "
-        << options_.signature_name();
-  }
-  for (const std::string& tag : cc->Outputs().GetTags()) {
-    RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
-        << "Can't find tag '" << tag << "' in signature "
-        << options_.signature_name();
-  }
-
-  {
-    absl::WriterMutexLock l(&mutex_);
-    inference_state_ = std::unique_ptr<InferenceState>();
-  }
-
-  if (options_.batch_size() == 1 || options_.batched_input()) {
-    cc->SetOffset(0);
-  }
-
-  return absl::OkStatus();
-}
-
-// Adds a batch dimension to the input tensor if specified in the calculator
-// options.
-absl::Status AddBatchDimension(tf::Tensor* input_tensor) {
-  if (options_.add_batch_dim_to_tensors()) {
-    tf::TensorShape new_shape(input_tensor->shape());
-    new_shape.InsertDim(0, 1);
-    RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape))
-        << "Could not add 0th dimension to tensor without changing its shape."
-        << " Current shape: " << input_tensor->shape().DebugString();
-  }
-  return absl::OkStatus();
-}
-
-absl::Status AggregateTensorPacket(
-    const std::string& tag_name, const Packet& packet,
-    std::map<Timestamp, std::map<std::string, tf::Tensor>>*
-        input_tensors_by_tag_by_timestamp,
-    InferenceState* inference_state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
-  tf::Tensor input_tensor(packet.Get<tf::Tensor>());
-  RET_CHECK_OK(AddBatchDimension(&input_tensor));
-  if (mediapipe::ContainsKey(recurrent_feed_tags_, tag_name)) {
-    // If we receive an input on a recurrent tag, override the state.
-    // It's OK to override the global state because there is just one
-    // input stream allowed for recurrent tensors.
-    inference_state_->input_tensor_batches_[tag_name].clear();
-  }
-  (*input_tensors_by_tag_by_timestamp)[packet.Timestamp()].insert(
-      std::make_pair(tag_name, input_tensor));
-  return absl::OkStatus();
-}
-
-// Removes the batch dimension of the output tensor if specified in the
-// calculator options.
-absl::Status RemoveBatchDimension(tf::Tensor* output_tensor) {
-  if (options_.add_batch_dim_to_tensors()) {
-    tf::TensorShape new_shape(output_tensor->shape());
-    new_shape.RemoveDim(0);
-    RET_CHECK(output_tensor->CopyFrom(*output_tensor, new_shape))
-        << "Could not remove 0th dimension from tensor without changing its "
-        << "shape. Current shape: " << output_tensor->shape().DebugString()
-        << " (The expected first dimension is 1 for a batch element.)";
-  }
-  return absl::OkStatus();
-}
-
-absl::Status Process(CalculatorContext* cc) override {
-  std::unique_ptr<InferenceState> inference_state_to_process;
-  {
-    absl::WriterMutexLock l(&mutex_);
-    if (inference_state_ == nullptr) {
-      inference_state_ = CreateInferenceState(cc);
     }
-    std::map<Timestamp, std::map<std::string, tf::Tensor>>
+    // A mediapipe::TensorFlowSession with a model loaded and ready for use.
-        input_tensors_by_tag_by_timestamp;
+    // For this calculator it must include a tag_to_tensor_map.
-    for (const std::string& tag_as_node_name : cc->Inputs().GetTags()) {
+    cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
-      if (cc->Inputs().Tag(tag_as_node_name).IsEmpty()) {
+    if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
-        // Recurrent tensors can be empty.
+      cc->InputSidePackets()
-        if (!mediapipe::ContainsKey(recurrent_feed_tags_, tag_as_node_name)) {
+          .Tag("RECURRENT_INIT_TENSORS")
-          if (options_.skip_on_missing_features()) {
+          .Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
-            return absl::OkStatus();
+    }
-          } else {
+    return absl::OkStatus();
-            return absl::InvalidArgumentError(absl::StrCat(
+  }
-                "Tag ", tag_as_node_name,
+
-                " not present at timestamp: ", cc->InputTimestamp().Value()));
+  std::unique_ptr<InferenceState> CreateInferenceState(CalculatorContext* cc)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+    std::unique_ptr<InferenceState> inference_state =
+        absl::make_unique<InferenceState>();
+    if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
+        !cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
+      std::map<std::string, tf::Tensor>* init_tensor_map;
+      init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
+          cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
+      for (const auto& p : *init_tensor_map) {
+        inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
+      }
+    }
+    return inference_state;
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
+
+    RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
+    session_ = cc->InputSidePackets()
+                   .Tag("SESSION")
+                   .Get<TensorFlowSession>()
+                   .session.get();
+    tag_to_tensor_map_ = cc->InputSidePackets()
+                             .Tag("SESSION")
+                             .Get<TensorFlowSession>()
+                             .tag_to_tensor_map;
+
+    // Validate and store the recurrent tags
+    RET_CHECK(options_.has_batch_size());
+    RET_CHECK(options_.batch_size() == 1 ||
+              options_.recurrent_tag_pair().empty())
+        << "To use recurrent_tag_pairs, batch_size must be 1.";
+    for (const auto& tag_pair : options_.recurrent_tag_pair()) {
+      const std::vector<std::string> tags = absl::StrSplit(tag_pair, ':');
+      RET_CHECK_EQ(tags.size(), 2)
+          << "recurrent_tag_pair must be a colon "
+             "separated std::string with two components: "
+          << tag_pair;
+      RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[0]))
+          << "Can't find tag '" << tags[0] << "' in signature "
+          << options_.signature_name();
+      RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[1]))
+          << "Can't find tag '" << tags[1] << "' in signature "
+          << options_.signature_name();
+      recurrent_feed_tags_.insert(tags[0]);
+      recurrent_fetch_tags_to_feed_tags_[tags[1]] = tags[0];
+    }
+
+    // Check that all tags are present in this signature bound to tensors.
+    for (const std::string& tag : cc->Inputs().GetTags()) {
+      RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
+          << "Can't find tag '" << tag << "' in signature "
+          << options_.signature_name();
+    }
+    for (const std::string& tag : cc->Outputs().GetTags()) {
+      RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag))
+          << "Can't find tag '" << tag << "' in signature "
+          << options_.signature_name();
+    }
+
+    {
+      absl::WriterMutexLock l(&mutex_);
+      inference_state_ = std::unique_ptr<InferenceState>();
+    }
+
+    if (options_.batch_size() == 1 || options_.batched_input()) {
+      cc->SetOffset(0);
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Adds a batch dimension to the input tensor if specified in the calculator
+  // options.
+  absl::Status AddBatchDimension(tf::Tensor* input_tensor) {
+    if (options_.add_batch_dim_to_tensors()) {
+      tf::TensorShape new_shape(input_tensor->shape());
+      new_shape.InsertDim(0, 1);
+      RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape))
+          << "Could not add 0th dimension to tensor without changing its shape."
+          << " Current shape: " << input_tensor->shape().DebugString();
+    }
+    return absl::OkStatus();
+  }
+
+  absl::Status AggregateTensorPacket(
+      const std::string& tag_name, const Packet& packet,
+      std::map<Timestamp, std::map<std::string, tf::Tensor>>*
+          input_tensors_by_tag_by_timestamp,
+      InferenceState* inference_state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
+    tf::Tensor input_tensor(packet.Get<tf::Tensor>());
+    RET_CHECK_OK(AddBatchDimension(&input_tensor));
+    if (mediapipe::ContainsKey(recurrent_feed_tags_, tag_name)) {
+      // If we receive an input on a recurrent tag, override the state.
+      // It's OK to override the global state because there is just one
+      // input stream allowed for recurrent tensors.
+      inference_state_->input_tensor_batches_[tag_name].clear();
+    }
+    (*input_tensors_by_tag_by_timestamp)[packet.Timestamp()].insert(
+        std::make_pair(tag_name, input_tensor));
+    return absl::OkStatus();
+  }
+
+  // Removes the batch dimension of the output tensor if specified in the
+  // calculator options.
+  absl::Status RemoveBatchDimension(tf::Tensor* output_tensor) {
+    if (options_.add_batch_dim_to_tensors()) {
+      tf::TensorShape new_shape(output_tensor->shape());
+      new_shape.RemoveDim(0);
+      RET_CHECK(output_tensor->CopyFrom(*output_tensor, new_shape))
+          << "Could not remove 0th dimension from tensor without changing its "
+          << "shape. Current shape: " << output_tensor->shape().DebugString()
+          << " (The expected first dimension is 1 for a batch element.)";
+    }
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    std::unique_ptr<InferenceState> inference_state_to_process;
+    {
+      absl::WriterMutexLock l(&mutex_);
+      if (inference_state_ == nullptr) {
+        inference_state_ = CreateInferenceState(cc);
+      }
+      std::map<Timestamp, std::map<std::string, tf::Tensor>>
+          input_tensors_by_tag_by_timestamp;
+      for (const std::string& tag_as_node_name : cc->Inputs().GetTags()) {
+        if (cc->Inputs().Tag(tag_as_node_name).IsEmpty()) {
+          // Recurrent tensors can be empty.
+          if (!mediapipe::ContainsKey(recurrent_feed_tags_, tag_as_node_name)) {
+            if (options_.skip_on_missing_features()) {
+              return absl::OkStatus();
+            } else {
+              return absl::InvalidArgumentError(absl::StrCat(
+                  "Tag ", tag_as_node_name,
+                  " not present at timestamp: ", cc->InputTimestamp().Value()));
+            }
           }
+        } else if (options_.batched_input()) {
+          const auto& tensor_packets =
+              cc->Inputs().Tag(tag_as_node_name).Get<std::vector<Packet>>();
+          if (tensor_packets.size() > options_.batch_size()) {
+            return absl::InvalidArgumentError(absl::StrCat(
+                "Batch for tag ", tag_as_node_name,
+                " has more packets than batch capacity. batch_size: ",
+                options_.batch_size(), " packets: ", tensor_packets.size()));
+          }
+          for (const auto& packet : tensor_packets) {
+            RET_CHECK_OK(AggregateTensorPacket(
+                tag_as_node_name, packet, &input_tensors_by_tag_by_timestamp,
+                inference_state_.get()));
+          }
+        } else {
+          RET_CHECK_OK(AggregateTensorPacket(
+              tag_as_node_name, cc->Inputs().Tag(tag_as_node_name).Value(),
+              &input_tensors_by_tag_by_timestamp, inference_state_.get()));
         }
-      } else if (options_.batched_input()) {
+      }
-        const auto& tensor_packets =
+      for (const auto& timestamp_and_input_tensors_by_tag :
-            cc->Inputs().Tag(tag_as_node_name).Get<std::vector<Packet>>();
+           input_tensors_by_tag_by_timestamp) {
-        if (tensor_packets.size() > options_.batch_size()) {
+        inference_state_->batch_timestamps_.emplace_back(
-          return absl::InvalidArgumentError(absl::StrCat(
+            timestamp_and_input_tensors_by_tag.first);
-              "Batch for tag ", tag_as_node_name,
+        for (const auto& input_tensor_and_tag :
-              " has more packets than batch capacity. batch_size: ",
+             timestamp_and_input_tensors_by_tag.second) {
-              options_.batch_size(), " packets: ", tensor_packets.size()));
+          inference_state_->input_tensor_batches_[input_tensor_and_tag.first]
+              .emplace_back(input_tensor_and_tag.second);
         }
-        for (const auto& packet : tensor_packets) {
+      }
-          RET_CHECK_OK(AggregateTensorPacket(tag_as_node_name, packet,
+      if (inference_state_->batch_timestamps_.size() == options_.batch_size() ||
-                                             &input_tensors_by_tag_by_timestamp,
+          options_.batched_input()) {
-                                             inference_state_.get()));
+        inference_state_to_process = std::move(inference_state_);
+        inference_state_ = std::unique_ptr<InferenceState>();
+      }
+    }
+
+    if (inference_state_to_process) {
+      MP_RETURN_IF_ERROR(
+          OutputBatch(cc, std::move(inference_state_to_process)));
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Close(CalculatorContext* cc) override {
+    std::unique_ptr<InferenceState> inference_state_to_process = nullptr;
+    {
+      absl::WriterMutexLock l(&mutex_);
+      if (cc->GraphStatus().ok() && inference_state_ != nullptr &&
+          !inference_state_->batch_timestamps_.empty()) {
+        inference_state_to_process = std::move(inference_state_);
+        inference_state_ = std::unique_ptr<InferenceState>();
+      }
+    }
+    if (inference_state_to_process) {
+      MP_RETURN_IF_ERROR(
+          OutputBatch(cc, std::move(inference_state_to_process)));
+    }
+    return absl::OkStatus();
+  }
+
+  // When a batch of input tensors is ready to be run, runs TensorFlow and
+  // outputs the output tensors. The output tensors have timestamps matching
+  // the input tensor that formed that batch element. Any requested
+  // batch_dimension is added and removed. This code takes advantage of the fact
+  // that copying a tensor shares the same reference-counted, heap allocated
+  // memory buffer. Therefore, copies are cheap and should not cause the memory
+  // buffer to fall out of scope. In contrast, concat is only used where
+  // necessary.
+  absl::Status OutputBatch(CalculatorContext* cc,
+                           std::unique_ptr<InferenceState> inference_state) {
+    const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
+    std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
+
+    for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
+      if (options_.batch_size() == 1) {
+        // Short circuit to avoid the cost of deep copying tensors in concat.
+        if (!keyed_tensors.second.empty()) {
+          input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
+                                     keyed_tensors.second[0]);
+        } else {
+          // The input buffer can be empty for recurrent tensors.
+          RET_CHECK(
+              mediapipe::ContainsKey(recurrent_feed_tags_, keyed_tensors.first))
+              << "A non-recurrent tensor does not have an input: "
+              << keyed_tensors.first;
         }
       } else {
-        RET_CHECK_OK(AggregateTensorPacket(
+        // Pad by replicating the first tens  or, then ignore the values.
-            tag_as_node_name, cc->Inputs().Tag(tag_as_node_name).Value(),
+        keyed_tensors.second.resize(options_.batch_size());
-            &input_tensors_by_tag_by_timestamp, inference_state_.get()));
+        std::fill(keyed_tensors.second.begin() +
-      }
+                      inference_state->batch_timestamps_.size(),
-    }
+                  keyed_tensors.second.end(), keyed_tensors.second[0]);
-    for (const auto& timestamp_and_input_tensors_by_tag :
+        tf::Tensor concated;
-         input_tensors_by_tag_by_timestamp) {
+        const tf::Status concat_status =
-      inference_state_->batch_timestamps_.emplace_back(
+            tf::tensor::Concat(keyed_tensors.second, &concated);
-          timestamp_and_input_tensors_by_tag.first);
+        CHECK(concat_status.ok()) << concat_status.ToString();
-      for (const auto& input_tensor_and_tag :
-           timestamp_and_input_tensors_by_tag.second) {
-        inference_state_->input_tensor_batches_[input_tensor_and_tag.first]
-            .emplace_back(input_tensor_and_tag.second);
-      }
-    }
-    if (inference_state_->batch_timestamps_.size() == options_.batch_size() ||
-        options_.batched_input()) {
-      inference_state_to_process = std::move(inference_state_);
-      inference_state_ = std::unique_ptr<InferenceState>();
-    }
-  }
-
-  if (inference_state_to_process) {
-    MP_RETURN_IF_ERROR(OutputBatch(cc, std::move(inference_state_to_process)));
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status Close(CalculatorContext* cc) override {
-  std::unique_ptr<InferenceState> inference_state_to_process = nullptr;
-  {
-    absl::WriterMutexLock l(&mutex_);
-    if (cc->GraphStatus().ok() && inference_state_ != nullptr &&
-        !inference_state_->batch_timestamps_.empty()) {
-      inference_state_to_process = std::move(inference_state_);
-      inference_state_ = std::unique_ptr<InferenceState>();
-    }
-  }
-  if (inference_state_to_process) {
-    MP_RETURN_IF_ERROR(OutputBatch(cc, std::move(inference_state_to_process)));
-  }
-  return absl::OkStatus();
-}
-
-// When a batch of input tensors is ready to be run, runs TensorFlow and
-// outputs the output tensors. The output tensors have timestamps matching
-// the input tensor that formed that batch element. Any requested
-// batch_dimension is added and removed. This code takes advantage of the fact
-// that copying a tensor shares the same reference-counted, heap allocated
-// memory buffer. Therefore, copies are cheap and should not cause the memory
-// buffer to fall out of scope. In contrast, concat is only used where
-// necessary.
-absl::Status OutputBatch(CalculatorContext* cc,
-                         std::unique_ptr<InferenceState> inference_state) {
-  const int64 start_time = absl::ToUnixMicros(clock_->TimeNow());
-  std::vector<std::pair<mediapipe::ProtoString, tf::Tensor>> input_tensors;
-
-  for (auto& keyed_tensors : inference_state->input_tensor_batches_) {
-    if (options_.batch_size() == 1) {
-      // Short circuit to avoid the cost of deep copying tensors in concat.
-      if (!keyed_tensors.second.empty()) {
         input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
-                                   keyed_tensors.second[0]);
+                                   concated);
-      } else {
-        // The input buffer can be empty for recurrent tensors.
-        RET_CHECK(
-            mediapipe::ContainsKey(recurrent_feed_tags_, keyed_tensors.first))
-            << "A non-recurrent tensor does not have an input: "
-            << keyed_tensors.first;
       }
-    } else {
-      // Pad by replicating the first tens  or, then ignore the values.
-      keyed_tensors.second.resize(options_.batch_size());
-      std::fill(keyed_tensors.second.begin() +
-                    inference_state->batch_timestamps_.size(),
-                keyed_tensors.second.end(), keyed_tensors.second[0]);
-      tf::Tensor concated;
-      const tf::Status concat_status =
-          tf::tensor::Concat(keyed_tensors.second, &concated);
-      CHECK(concat_status.ok()) << concat_status.ToString();
-      input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
-                                 concated);
     }
-  }
+    inference_state->input_tensor_batches_.clear();
-  inference_state->input_tensor_batches_.clear();
+    std::vector<mediapipe::ProtoString> output_tensor_names;
-  std::vector<mediapipe::ProtoString> output_tensor_names;
+    std::vector<std::string> output_name_in_signature;
-  std::vector<std::string> output_name_in_signature;
+    for (const std::string& tag : cc->Outputs().GetTags()) {
-  for (const std::string& tag : cc->Outputs().GetTags()) {
+      output_tensor_names.emplace_back(tag_to_tensor_map_[tag]);
-    output_tensor_names.emplace_back(tag_to_tensor_map_[tag]);
+      output_name_in_signature.emplace_back(tag);
-    output_name_in_signature.emplace_back(tag);
-  }
-  for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
-    // Ensure that we always fetch the recurrent state tensors.
-    if (std::find(output_name_in_signature.begin(),
-                  output_name_in_signature.end(),
-                  tag_pair.first) == output_name_in_signature.end()) {
-      output_tensor_names.emplace_back(tag_to_tensor_map_[tag_pair.first]);
-      output_name_in_signature.emplace_back(tag_pair.first);
     }
-  }
+    for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
-  std::vector<tf::Tensor> outputs;
+      // Ensure that we always fetch the recurrent state tensors.
+      if (std::find(output_name_in_signature.begin(),
+                    output_name_in_signature.end(),
+                    tag_pair.first) == output_name_in_signature.end()) {
+        output_tensor_names.emplace_back(tag_to_tensor_map_[tag_pair.first]);
+        output_name_in_signature.emplace_back(tag_pair.first);
+      }
+    }
+    std::vector<tf::Tensor> outputs;
 
-  SimpleSemaphore* session_run_throttle = nullptr;
+    SimpleSemaphore* session_run_throttle = nullptr;
-  if (options_.max_concurrent_session_runs() > 0) {
+    if (options_.max_concurrent_session_runs() > 0) {
-    session_run_throttle =
+      session_run_throttle =
-        get_session_run_throttle(options_.max_concurrent_session_runs());
+          get_session_run_throttle(options_.max_concurrent_session_runs());
-    session_run_throttle->Acquire(1);
+      session_run_throttle->Acquire(1);
-  }
+    }
-  const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
+    const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
-  tf::Status tf_status;
+    tf::Status tf_status;
-  {
+    {
 #if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__)
-    tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
+      tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
 #endif
-    tf_status = session_->Run(input_tensors, output_tensor_names,
+      tf_status = session_->Run(input_tensors, output_tensor_names,
-                              {} /* target_node_names */, &outputs);
+                                {} /* target_node_names */, &outputs);
-  }
+    }
 
-  if (session_run_throttle != nullptr) {
+    if (session_run_throttle != nullptr) {
-    session_run_throttle->Release(1);
+      session_run_throttle->Release(1);
-  }
+    }
 
-  // RET_CHECK on the tf::Status object itself in order to print an
+    // RET_CHECK on the tf::Status object itself in order to print an
-  // informative error message.
+    // informative error message.
-  RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
+    RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
 
-  const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
+    const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow());
-  cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
+    cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix)
-      ->IncrementBy(run_end_time - run_start_time);
+        ->IncrementBy(run_end_time - run_start_time);
-  cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
+    cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment();
 
-  // Feed back the recurrent state.
+    // Feed back the recurrent state.
-  for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
+    for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) {
-    int pos = std::find(output_name_in_signature.begin(),
+      int pos = std::find(output_name_in_signature.begin(),
-                        output_name_in_signature.end(), tag_pair.first) -
+                          output_name_in_signature.end(), tag_pair.first) -
-              output_name_in_signature.begin();
+                output_name_in_signature.begin();
-    inference_state->input_tensor_batches_[tag_pair.second].emplace_back(
+      inference_state->input_tensor_batches_[tag_pair.second].emplace_back(
-        outputs[pos]);
+          outputs[pos]);
-  }
+    }
 
-  absl::WriterMutexLock l(&mutex_);
+    absl::WriterMutexLock l(&mutex_);
-  // Set that we want to split on each index of the 0th dimension.
+    // Set that we want to split on each index of the 0th dimension.
-  std::vector<tf::int64> split_vector(options_.batch_size(), 1);
+    std::vector<tf::int64> split_vector(options_.batch_size(), 1);
-  for (int i = 0; i < output_tensor_names.size(); ++i) {
+    for (int i = 0; i < output_tensor_names.size(); ++i) {
-    if (options_.batch_size() == 1) {
+      if (options_.batch_size() == 1) {
-      if (cc->Outputs().HasTag(output_name_in_signature[i])) {
+        if (cc->Outputs().HasTag(output_name_in_signature[i])) {
-        tf::Tensor output_tensor(outputs[i]);
+          tf::Tensor output_tensor(outputs[i]);
-        RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
+          RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
-        cc->Outputs()
+          cc->Outputs()
-            .Tag(output_name_in_signature[i])
+              .Tag(output_name_in_signature[i])
-            .Add(new tf::Tensor(output_tensor),
+              .Add(new tf::Tensor(output_tensor),
-                 inference_state->batch_timestamps_[0]);
+                   inference_state->batch_timestamps_[0]);
-      }
+        }
-    } else {
+      } else {
-      std::vector<tf::Tensor> split_tensors;
+        std::vector<tf::Tensor> split_tensors;
-      const tf::Status split_status =
+        const tf::Status split_status =
-          tf::tensor::Split(outputs[i], split_vector, &split_tensors);
+            tf::tensor::Split(outputs[i], split_vector, &split_tensors);
-      CHECK(split_status.ok()) << split_status.ToString();
+        CHECK(split_status.ok()) << split_status.ToString();
-      // Loop over timestamps so that we don't copy the padding.
+        // Loop over timestamps so that we don't copy the padding.
-      for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) {
+        for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) {
-        tf::Tensor output_tensor(split_tensors[j]);
+          tf::Tensor output_tensor(split_tensors[j]);
-        RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
+          RET_CHECK_OK(RemoveBatchDimension(&output_tensor));
-        cc->Outputs()
+          cc->Outputs()
-            .Tag(output_name_in_signature[i])
+              .Tag(output_name_in_signature[i])
-            .Add(new tf::Tensor(output_tensor),
+              .Add(new tf::Tensor(output_tensor),
-                 inference_state->batch_timestamps_[j]);
+                   inference_state->batch_timestamps_[j]);
+        }
       }
     }
+
+    // Get end time and report.
+    const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
+    cc->GetCounter(kTotalUsecsCounterSuffix)
+        ->IncrementBy(end_time - start_time);
+    cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
+        ->IncrementBy(inference_state->batch_timestamps_.size());
+
+    // Make sure we hold on to the recursive state.
+    if (!options_.recurrent_tag_pair().empty()) {
+      inference_state_ = std::move(inference_state);
+      inference_state_->batch_timestamps_.clear();
+    }
+
+    return absl::OkStatus();
   }
 
-  // Get end time and report.
+ private:
-  const int64 end_time = absl::ToUnixMicros(clock_->TimeNow());
+  // The Session object is provided by a packet factory and is owned by the
-  cc->GetCounter(kTotalUsecsCounterSuffix)->IncrementBy(end_time - start_time);
+  // MediaPipe framework. Individual calls are thread-safe, but session state
-  cc->GetCounter(kTotalProcessedTimestampsCounterSuffix)
+  // may be shared across threads.
-      ->IncrementBy(inference_state->batch_timestamps_.size());
+  tf::Session* session_;
 
-  // Make sure we hold on to the recursive state.
+  // A mapping between stream tags and the tensor names they are bound to.
-  if (!options_.recurrent_tag_pair().empty()) {
+  std::map<std::string, std::string> tag_to_tensor_map_;
-    inference_state_ = std::move(inference_state);
+
-    inference_state_->batch_timestamps_.clear();
+  absl::Mutex mutex_;
+  std::unique_ptr<InferenceState> inference_state_ ABSL_GUARDED_BY(mutex_);
+
+  // The options for the calculator.
+  TensorFlowInferenceCalculatorOptions options_;
+
+  // Store the feed and fetch tags for feed/fetch recurrent networks.
+  std::set<std::string> recurrent_feed_tags_;
+  std::map<std::string, std::string> recurrent_fetch_tags_to_feed_tags_;
+
+  // Clock used to measure the computation time in OutputBatch().
+  std::unique_ptr<mediapipe::Clock> clock_;
+
+  // The static singleton semaphore to throttle concurrent session runs.
+  static SimpleSemaphore* get_session_run_throttle(
+      int32 max_concurrent_session_runs) {
+    static SimpleSemaphore* session_run_throttle =
+        new SimpleSemaphore(max_concurrent_session_runs);
+    return session_run_throttle;
   }
-
+};
-  return absl::OkStatus();
-}
-
-private:
-// The Session object is provided by a packet factory and is owned by the
-// MediaPipe framework. Individual calls are thread-safe, but session state may
-// be shared across threads.
-tf::Session* session_;
-
-// A mapping between stream tags and the tensor names they are bound to.
-std::map<std::string, std::string> tag_to_tensor_map_;
-
-absl::Mutex mutex_;
-std::unique_ptr<InferenceState> inference_state_ ABSL_GUARDED_BY(mutex_);
-
-// The options for the calculator.
-TensorFlowInferenceCalculatorOptions options_;
-
-// Store the feed and fetch tags for feed/fetch recurrent networks.
-std::set<std::string> recurrent_feed_tags_;
-std::map<std::string, std::string> recurrent_fetch_tags_to_feed_tags_;
-
-// Clock used to measure the computation time in OutputBatch().
-std::unique_ptr<mediapipe::Clock> clock_;
-
-// The static singleton semaphore to throttle concurrent session runs.
-static SimpleSemaphore* get_session_run_throttle(
-    int32 max_concurrent_session_runs) {
-  static SimpleSemaphore* session_run_throttle =
-      new SimpleSemaphore(max_concurrent_session_runs);
-  return session_run_throttle;
-}
-}
-;
 REGISTER_CALCULATOR(TensorFlowInferenceCalculator);
 
 constexpr char TensorFlowInferenceCalculator::kTotalUsecsCounterSuffix[];

mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc

@@ -16,12 +16,12 @@
 #include <string>
 #include <vector>
 
+#include "absl/flags/flag.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
@@ -19,7 +20,6 @@
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"