Project import generated by Copybara.

GitOrigin-RevId: 6a704ded0bf489614797082e7e7cda1068477ef5

Dockerfile

@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.4.1
+ARG BAZEL_VERSION=3.7.2
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

MANIFEST.in

@@ -10,3 +10,7 @@ include requirements.txt
 recursive-include mediapipe/modules *.tflite *.txt *.binarypb
 exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
 exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite

README.md

@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

WORKSPACE

@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
+    urls = [
-    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+    ],
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
 )
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "3.4.0")
+versions.check(minimum_bazel_version = "3.7.2")
-
 
 # ABSL cpp library lts_2020_09_23
 http_archive(
@@ -38,8 +41,8 @@ http_archive(
 
 http_archive(
    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-main",
+   strip_prefix = "rules_foreign_cc-0.1.0",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
+   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -117,7 +120,8 @@ http_archive(
 # libyuv
 http_archive(
     name = "libyuv",
-    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
+    # Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8.
+    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"],
     build_file = "@//third_party:libyuv.BUILD",
 )
 
@@ -304,8 +308,8 @@ http_archive(
 
 # Maven dependencies.
 
-RULES_JVM_EXTERNAL_TAG = "3.2"
+RULES_JVM_EXTERNAL_TAG = "4.0"
-RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
 
 http_archive(
     name = "rules_jvm_external",
@@ -318,7 +322,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 
 # Important: there can only be one maven_install rule. Add new maven deps here.
 maven_install(
-    name = "maven",
     artifacts = [
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.2.0",
@@ -343,10 +346,10 @@ maven_install(
         "org.hamcrest:hamcrest-library:1.3",
     ],
     repositories = [
-        "https://jcenter.bintray.com",
         "https://maven.google.com",
         "https://dl.google.com/dl/android/maven2",
         "https://repo1.maven.org/maven2",
+        "https://jcenter.bintray.com",
     ],
     fetch_sources = True,
     version_conflict_policy = "pinned",
@@ -363,10 +366,10 @@ http_archive(
     ],
 )
 
-#Tensorflow repo should always go after the other external dependencies.
+# Tensorflow repo should always go after the other external dependencies.
-# 2020-12-09
+# 2021-03-25
-_TENSORFLOW_GIT_COMMIT = "0eadbb13cef1226b1bae17c941f7870734d97f8a"
+_TENSORFLOW_GIT_COMMIT = "c67f68021824410ebe9f18513b8856ac1c6d4887"
-_TENSORFLOW_SHA256= "4ae06daa5b09c62f31b7bc1f781fd59053f286dd64355830d8c2ac601b795ef0"
+_TENSORFLOW_SHA256= "fd07d0b39422dc435e268c5e53b2646a8b4b1e3151b87837b43f86068faae87f"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -383,5 +386,7 @@ http_archive(
     sha256 = _TENSORFLOW_SHA256,
 )
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
-tf_workspace(tf_repo_name = "org_tensorflow")
+tf_workspace3()
+load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+tf_workspace2()

docs/framework_concepts/packets.md

@@ -12,19 +12,30 @@ nav_order: 3
 {:toc}
 ---
 
-Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
+Calculators communicate by sending and receiving packets. Typically a single
+packet is sent along each input stream at each input timestamp. A packet can
+contain any kind of data, such as a single frame of video or a single integer
+detection count.
 
 ## Creating a packet
 
-Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
+Packets are generally created with `mediapipe::MakePacket<T>()` or
+`mediapipe::Adopt()` (from packet.h).
 
 ```c++
-// Create some data.
+// Create a packet containing some new data.
-auto data = absl::make_unique<MyDataClass>("constructor_argument");
+Packet p = MakePacket<MyDataClass>("constructor_argument");
-// Create a packet to own the data.
-Packet p = Adopt(data.release());
 // Make a new packet with the same data and a different timestamp.
 Packet p2 = p.At(Timestamp::PostStream());
 ```
 
+or:
+
+```c++
+// Create some new data.
+auto data = absl::make_unique<MyDataClass>("constructor_argument");
+// Create a packet to own the data.
+Packet p = Adopt(data.release()).At(Timestamp::PostStream());
+```
+
 Data within a packet is accessed with `Packet::Get<T>()`

docs/getting_started/android.md

@@ -28,7 +28,7 @@ Gradle.
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
 *   Setup Android SDK release 28.0.3 and above.
-*   Setup Android NDK r18b and above.
+*   Setup Android NDK version between 18 and 21.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
 below for Android Studio setup). However, if you prefer using MediaPipe without

docs/getting_started/install.md

@@ -25,25 +25,11 @@ install --user six`.
 
 ## Installing on Debian and Ubuntu
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
-
-    For Nvidia Jetson and Raspberry Pi devices with aarch64 Linux, Bazel needs
-    to be built from source:
-
-    ```bash
-    # For Bazel 3.4.1
-    mkdir $HOME/bazel-3.4.1
-    cd $HOME/bazel-3.4.1
-    wget https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-dist.zip
-    sudo apt-get install build-essential openjdk-8-jdk python zip unzip
-    unzip bazel-3.4.1-dist.zip
-    env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
-    sudo cp output/bazel /usr/local/bin/
-    ```
 
 2.  Checkout MediaPipe repository.
 
@@ -207,11 +193,11 @@ build issues.
 
 **Disclaimer**: Running MediaPipe on CentOS is experimental.
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -336,11 +322,11 @@ build issues.
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
         Tools by `xcode-select --install`.
 
-2.  Install Bazel.
+2.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    to install Bazel 3.4 or higher.
+    to install Bazelisk.
 
 3.  Checkout MediaPipe repository.
 
@@ -353,7 +339,7 @@ build issues.
 4.  Install OpenCV and FFmpeg.
 
     Option 1. Use HomeBrew package manager tool to install the pre-compiled
-    OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
+    OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
 
     ```bash
     $ brew install opencv@3
@@ -484,29 +470,36 @@ next section.
 
 4.  Install Visual C++ Build Tools 2019 and WinSDK
 
-    Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
+    Go to
-    build tools, and install Microsoft Visual C++ 2019 Redistributable and
+    [the VisualStudio website](ttps://visualstudio.microsoft.com/visual-cpp-build-tools),
-    Microsoft Build Tools 2019.
+    download build tools, and install Microsoft Visual C++ 2019 Redistributable
+    and Microsoft Build Tools 2019.
 
     Download the WinSDK from
-    https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
+    [the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
-    install.
+    and install.
 
-5.  Install Bazel and add the location of the Bazel executable to the `%PATH%`
+5.  Install Bazel or Bazelisk and add the location of the Bazel executable to
-    environment variable.
+    the `%PATH%` environment variable.
 
-    Follow the official
+    Option 1. Follow
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
+    [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.4 or higher.
+    to install Bazel 3.7.2 or higher.
 
-6.  Set Bazel variables.
+    Option 2. Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
+
+6.  Set Bazel variables. Learn more details about
+    ["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
+    in the Bazel official documentation.
 
     ```
-    # Find the exact paths and version numbers from your local version.
+    # Please find the exact paths and version numbers from your local version.
     C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
     C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
-    C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
+    C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
-    C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
+    C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
     ```
 
 7.  Checkout MediaPipe repository.
@@ -593,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel.
+5.  Install Bazelisk.
 
-    ```bash
+    Follow the official
-    username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
-    https://storage.googleapis.com/bazel/3.4.1/release/bazel-3.4.1-installer-linux-x86_64.sh && \
+    to install Bazelisk.
-    sudo mkdir -p /usr/local/bazel/3.4.1 && \
-    chmod 755 bazel-3.4.1-installer-linux-x86_64.sh && \
-    sudo ./bazel-3.4.1-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.1 && \
-    source /usr/local/bazel/3.4.1/lib/bazel/bin/bazel-complete.bash
-
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.1/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/3.4.1/lib/bazel/bin/bazel'
-    ```
 
 6.  Checkout MediaPipe repository.
 

docs/getting_started/javascript.md

@@ -72,9 +72,9 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 [Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
 [H-npm]: https://www.npmjs.com/package/@mediapipe/hands
 [P-npm]: https://www.npmjs.com/package/@mediapipe/pose
-[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
-[cam-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
-[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
 [Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
 [F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
 [Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection

docs/index.md

@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/solutions/face_detection.md

@@ -78,6 +78,7 @@ Supported configuration options:
 import cv2
 import mediapipe as mp
 mp_face_detction = mp.solutions.face_detection
+mp_drawing = mp.solutions.drawing_utils
 
 # For static images:
 with mp_face_detection.FaceDetection(
@@ -183,8 +184,8 @@ function onResults(results) {
   canvasCtx.restore();
 }
 
-const faceDetection = new Objectron({locateFile: (file) => {
+const faceDetection = new FaceDetection({locateFile: (file) => {
-  return `https://cdn.jsdelivr.net/npm/@mediapipe/objectr[email protected]/${file}`;
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detecti[email protected]/${file}`;
 }});
 faceDetection.setOptions({
   minDetectionConfidence: 0.5

docs/solutions/objectron.md

@@ -358,15 +358,17 @@ cap.release()
 ## Example Apps
 
 Please first see general instructions for
-[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
+[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
-how to build MediaPipe examples.
+[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
 
 Note: To visualize a graph, copy the graph and paste it into
 [MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Two-stage Objectron
+### Mobile
+
+#### Two-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -404,7 +406,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Single-stage Objectron
+#### Single-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -428,7 +430,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Assets
+#### Assets
 
 Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
  format into a custom .uuu format. This can be done for user assets as follows:
@@ -449,9 +451,35 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
 > absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
 
-### Coordinate Systems
 
-#### Object Coordinate
+### Desktop
+
+To build the application, run:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
+```
+
+To run the application, replace `<input video path>` and `<output video path>`
+in the command below with your own paths, and `<landmark model path>` and
+`<allowed labels>` with the following:
+
+Category | `<landmark model path>`                                                     | `<allowed labels>`
+:------- | :-------------------------------------------------------------------------- | :-----------------
+Shoe     | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
+Chair    | mediapipe/modules/objectron/object_detection_3d_chair.tflite    | Chair
+Cup      | mediapipe/modules/objectron/object_detection_3d_cup.tflite      | Mug
+Camera   | mediapipe/modules/objectron/object_detection_3d_camera.tflite   | Camera
+
+```
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
+  --input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
+```
+
+## Coordinate Systems
+
+### Object Coordinate
 
 Each object has its object coordinate frame. We use the below object coordinate
 definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
@@ -459,7 +487,7 @@ origin is at the center of the 3D bounding box.
 
 ![box_coordinate.svg](../images/box_coordinate.svg)
 
-#### Camera Coordinate
+### Camera Coordinate
 
 A 3D object is parameterized by its `scale` and `rotation`, `translation` with
 regard to the camera coordinate frame. In this API we use the below camera
@@ -476,7 +504,7 @@ camera frame by applying `rotation` and `translation`:
 landmarks_3d = rotation * scale * unit_box + translation
 ```
 
-#### NDC Space
+### NDC Space
 
 In this API we use
 [NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
@@ -495,7 +523,7 @@ y_ndc = -fy * Y / Z + py
 z_ndc = 1 / Z
 ```
 
-#### Pixel Space
+### Pixel Space
 
 In this API we set upper-left coner of an image as the origin of pixel
 coordinate. One can convert from NDC to pixel space as follows:
@@ -532,10 +560,11 @@ py = -py_pixel * 2.0 / image_height + 1.0
     [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
 *   Google AI Blog:
     [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
+*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
 *   Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
     Shape Supervision](https://arxiv.org/abs/2003.03522)
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
-    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
+    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
 *   [Models and model cards](./models.md#objectron)
 *   [Python Colab](https://mediapipe.page.link/objectron_py_colab)

docs/solutions/pose_classification.md

@@ -25,10 +25,11 @@ One of the applications
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 can enable is fitness. More specifically - pose classification and repetition
 counting. In this section we'll provide basic guidance on building a custom pose
-classifier with the help of [Colabs](#colabs) and wrap it in a simple
+classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
-[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
+demo within
-powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
-are used for demonstration purposes as the most common exercises.
+Push-ups and squats are used for demonstration purposes as the most common
+exercises.
 
 ![pose_classification_pushups_and_squats.gif](../images/mobile/pose_classification_pushups_and_squats.gif) |
 :--------------------------------------------------------------------------------------------------------: |
@@ -47,7 +48,7 @@ determines the object's class based on the closest samples in the training set.
     classifier and form a training set using these [Colabs](#colabs),
 3.  Perform the classification itself followed by repetition counting (e.g., in
     the
-    [ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)).
+    [ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
 
 ## Training Set
 
@@ -76,7 +77,7 @@ video right in the Colab.
 
 Code of the classifier is available both in the
 [`Pose Classification Colab (Extended)`] and in the
-[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
 Please refer to them for details of the approach described below.
 
 The k-NN algorithm used for pose classification requires a feature vector
@@ -127,11 +128,13 @@ where the pose class and the counter can't be changed.
 
 ## Future Work
 
-We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
+We are actively working on improving
-allow us to use joint angles in the feature vectors, which are more natural and
+[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
-easier to configure (although distances can still be useful to detect touches
+prediction. It will allow us to use joint angles in the feature vectors, which
-between body parts) and to perform rotation normalization of poses and reduce
+are more natural and easier to configure (although distances can still be useful
-the number of camera angles required for accurate k-NN classification.
+to detect touches between body parts) and to perform rotation normalization of
+poses and reduce the number of camera angles required for accurate k-NN
+classification.
 
 ## Colabs
 

docs/solutions/solutions.md

@@ -28,7 +28,7 @@ has_toc: false
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                        | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/tools/tracing_and_profiling.md

@@ -41,6 +41,7 @@ profiler_config {
   trace_enabled: true
   enable_profiler: true
   trace_log_interval_count: 200
+  trace_log_path: "/sdcard/Download/"
 }
 ```
 
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
 
 *   On the desktop, this will be the `/tmp` directory.
 
-*   On Android, this will be the `/sdcard` directory.
+*   On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
 
 *   On iOS, this can be reached through XCode. Select "Window/Devices and
     Simulators" and select the "Devices" tab.
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     *   Include the line below in your `AndroidManifest.xml` file.
 
         ```xml
-        <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+        <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
         ```
 
     *   Grant the permission either upon first app launch, or by going into
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
     events to a trace log files at:
 
     ```bash
-    /sdcard/mediapipe_trace_0.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_0.binarypb
-    /sdcard/mediapipe_trace_1.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_1.binarypb
     ```
 
     After every 5 sec, writing shifts to a successive trace log file, such that
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
     trace files have been written to the device using adb shell.
 
     ```bash
-    adb shell "ls -la /sdcard/"
+    adb shell "ls -la /storage/emulated/0/Download"
     ```
 
-    On android, MediaPipe selects the external storage directory `/sdcard` for
+    On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
     trace logs. This directory can be overridden using the setting
     `trace_log_path`, like:
 
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     profiler_config {
       trace_enabled: true
       enable_profiler: true
-      trace_log_path: "/sdcard/profiles/"
+      trace_log_path: "/sdcard/Download/profiles/"
     }
     ```
 
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
 
     ```bash
     # from your terminal
-    adb pull /sdcard/mediapipe_trace_0.binarypb
+    adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
     # if successful you should see something like
     # /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
     ```

mediapipe/calculators/audio/BUILD

@@ -128,7 +128,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -147,7 +147,7 @@ cc_library(
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp/mfcc",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -168,7 +168,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:resampler",
         "@com_google_audio_tools//audio/dsp:resampler_q",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -208,7 +208,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@com_google_audio_tools//audio/dsp/spectrogram",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -228,7 +228,7 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -242,9 +242,9 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -261,7 +261,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -276,7 +276,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -296,7 +296,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:number_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -314,7 +314,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -333,7 +333,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -352,6 +352,6 @@ cc_test(
         "//mediapipe/framework/tool:validate_type",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:signal_vector_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
@@ -25,7 +25,7 @@ namespace mediapipe {
 
 TEST(AudioDecoderCalculatorTest, TestWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -34,7 +34,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -56,7 +56,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
 
 TEST(AudioDecoderCalculatorTest, Test48KWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -65,7 +65,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -87,7 +87,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
 
 TEST(AudioDecoderCalculatorTest, TestMP3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -96,7 +96,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -118,7 +118,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
 
 TEST(AudioDecoderCalculatorTest, TestAAC) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -127,7 +127,7 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",

mediapipe/calculators/core/BUILD

@@ -414,7 +414,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -430,7 +430,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -450,6 +450,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "nonzero_calculator",
+    srcs = ["nonzero_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "mux_calculator_test",
     srcs = ["mux_calculator_test.cc"],
@@ -776,7 +790,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -793,7 +807,7 @@ cc_test(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -1024,7 +1038,7 @@ cc_library(
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/memory",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -70,7 +70,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           node {
@@ -90,7 +90,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:ints_plus_one"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));
@@ -197,7 +197,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "force_ints_to_be_timestamp_bound_update"
@@ -229,7 +229,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
             input_stream: "ints_plus_one"
             output_stream: "ints_plus_one_passed_through"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one_passed_through", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
@@ -338,7 +338,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "multiplier"
@@ -362,7 +362,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:multiplied_ints"
           }
-        )");
+        )pb");
     tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -38,14 +38,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
 
 TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {};
@@ -60,14 +60,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {0, 1, 2, 3};
@@ -85,14 +85,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   {
@@ -133,7 +133,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_vector"
         node {
           calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
@@ -143,7 +143,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
             [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("output_vector", &graph_config, &outputs);
@@ -178,7 +178,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         input_side_packet: "max_vec_size"
@@ -186,7 +186,7 @@ TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   // This should override the default of 1 set in the options.
   runner.MutableSidePackets()->Index(0) = Adopt(new int(2));

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -392,7 +392,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         input_stream: "in_3"
@@ -403,7 +403,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
           input_stream: "in_3"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -456,7 +456,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -465,7 +465,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
           input_stream: "in_2"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -505,7 +505,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -519,7 +519,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
             }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -82,7 +82,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -111,7 +111,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
 
 TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -131,7 +131,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -152,7 +152,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:int_packet"
@@ -163,14 +163,14 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -181,7 +181,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             max_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -51,7 +51,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -61,7 +61,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -76,7 +76,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -86,7 +86,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -101,7 +101,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -111,7 +111,7 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
             min_quantized_value: -2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -57,7 +57,7 @@ namespace mediapipe {
 //
 // The "ALLOW" stream indicates the transition between accepting frames and
 // dropping frames.  "ALLOW = true" indicates the start of accepting frames
-// including the current timestamp, and "ALLOW = true" indicates the start of
+// including the current timestamp, and "ALLOW = false" indicates the start of
 // dropping frames including the current timestamp.
 //
 // FlowLimiterCalculator provides limited support for multiple input streams.

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -126,7 +126,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The LambdaCalculator is used to keep certain frames in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -143,7 +143,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
         input_stream: 'in_1_sampled'
         output_stream: 'out_1'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -271,7 +271,7 @@ REGISTER_CALCULATOR(DropCalculator);
 class FlowLimiterCalculatorTest : public testing::Test {
  protected:
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -296,7 +296,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
         input_stream: 'PACKET:out_1_sampled'
         output_stream: 'PACKET:out_1'
       }
-    )");
+    )pb");
   }
 
   // Parse an absl::Time from RFC3339 format.
@@ -348,10 +348,10 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -419,11 +419,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -483,11 +483,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -548,7 +548,7 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -576,13 +576,13 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -657,7 +657,7 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -685,13 +685,13 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 0
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -64,13 +64,13 @@ const char kMatrixText2[] =
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_side_packet: "MINUEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(
@@ -81,12 +81,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig) {
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_stream: "SUBTRAHEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
@@ -95,12 +95,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "MINUEND:input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);
@@ -124,12 +124,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "SUBTRAHEND:input_matrix"
         input_side_packet: "MINUEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);

mediapipe/calculators/core/merge_calculator_test.cc

@@ -26,31 +26,33 @@ namespace {
 
 // Checks that the calculator fails if no input streams are provided.
 TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     output_stream: "merged_output"
-  )"));
+  )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner.Run().ok());
 }
 
 // Checks that the calculator fails with an incorrect number of output streams.
 TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
-  CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner1(
-    calculator: "MergeCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "input1"
+        calculator: "MergeCalculator"
-    input_stream: "input2"
+        input_stream: "input1"
-  )"));
+        input_stream: "input2"
+      )pb"));
   // Expect calculator to fail.
   EXPECT_FALSE(runner1.Run().ok());
 
-  CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner2(
-    calculator: "MergeCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "input1"
+        calculator: "MergeCalculator"
-    input_stream: "input2"
+        input_stream: "input1"
-    output_stream: "output1"
+        input_stream: "input2"
-    output_stream: "output2"
+        output_stream: "output1"
-  )"));
+        output_stream: "output2"
+      )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner2.Run().ok());
 }
@@ -58,12 +60,12 @@ TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
 // Ensures two streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingTwoStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integers 10, 20, 30, occurring at times 10, 20, 30.
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -102,13 +104,13 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
 // Ensures three streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingThreeStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     input_stream: "input3"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integer 30 occurring at time 30.
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/core/mux_calculator_test.cc

@@ -31,7 +31,7 @@ namespace {
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input stream. All MuxCalculator inputs are present at each timestamp.
-constexpr char kTestGraphConfig1[] = R"proto(
+constexpr char kTestGraphConfig1[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -60,12 +60,12 @@ constexpr char kTestGraphConfig1[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input side packet. All MuxCalculator inputs are present at each
 // timestamp.
-constexpr char kTestGraphConfig2[] = R"proto(
+constexpr char kTestGraphConfig2[] = R"pb(
   input_side_packet: "input_selector"
   input_stream: "input"
   output_stream: "test_output"
@@ -93,12 +93,12 @@ constexpr char kTestGraphConfig2[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with mux input stream handler, and the input selection is driven
 // by an input stream. Only one MuxCalculator input is present at each
 // timestamp.
-constexpr char kTestGraphConfig3[] = R"proto(
+constexpr char kTestGraphConfig3[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -117,7 +117,7 @@ constexpr char kTestGraphConfig3[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 constexpr char kOutputName[] = "test_output";
 constexpr char kInputName[] = "input";
@@ -235,7 +235,7 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
   EXPECT_EQ(output, input_packets);
 }
 
-constexpr char kDualInputGraphConfig[] = R"proto(
+constexpr char kDualInputGraphConfig[] = R"pb(
   input_stream: "input_0"
   input_stream: "input_1"
   input_stream: "input_select"
@@ -247,7 +247,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
   CalculatorGraphConfig config =

mediapipe/calculators/core/nonzero_calculator.cc

@@ -0,0 +1,42 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
+class NonZeroCalculator : public Node {
+ public:
+  static constexpr Input<int>::SideFallback kIn{"INPUT"};
+  static constexpr Output<int> kOut{"OUTPUT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      auto output = std::make_unique<int>((*kIn(cc) != 0) ? 1 : 0);
+      kOut(cc).Send(std::move(output));
+    }
+    return absl::OkStatus();
+  }
+};
+
+MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/packet_presence_calculator_test.cc

@@ -40,7 +40,7 @@ MATCHER_P2(BoolPacket, value, timestamp, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> output_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'allow'
         input_stream: 'value'
         node {
@@ -54,7 +54,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'PACKET:gated_value'
           output_stream: 'PRESENCE:presence'
         }
-      )");
+      )pb");
   tool::AddVectorSink("presence", &graph_config, &output_packets);
 
   CalculatorGraph graph;

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
-  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     output_data_id_ = cc->Outputs().GetId("", 0);
   }
 
-  period_count_ = 0;
   frame_rate_ = resampler_options.frame_rate();
-  base_timestamp_ = resampler_options.has_base_timestamp()
-                        ? Timestamp(resampler_options.base_timestamp())
-                        : Timestamp::Unset();
   start_time_ = resampler_options.has_start_time()
                     ? Timestamp(resampler_options.start_time())
                     : Timestamp::Min();
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     }
   }
 
-  if (jitter_ != 0.0) {
+  strategy_ = GetSamplingStrategy(resampler_options);
-    if (resampler_options.output_header() !=
+
-        PacketResamplerCalculatorOptions::NONE) {
+  return strategy_->Open(cc);
-      LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
-                      "the actual value.";
-    }
-    if (flush_last_packet_) {
-      flush_last_packet_ = false;
-      LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
-                      "ignored, because we are adding jitter.";
-    }
-    const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
-    random_ = CreateSecureRandom(seed);
-    if (random_ == nullptr) {
-      return absl::Status(
-          absl::StatusCode::kInvalidArgument,
-          "SecureRandom is not available.  With \"jitter\" specified, "
-          "PacketResamplerCalculator processing cannot proceed.");
-    }
-    packet_reservoir_random_ = CreateSecureRandom(seed);
-  }
-  packet_reservoir_ =
-      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
-  return absl::OkStatus();
 }
 
 absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
       return absl::OkStatus();
     }
   }
-  if (jitter_ != 0.0 && random_ != nullptr) {
+
-    // Packet reservior is used to make sure there's an output for every period,
+  if (absl::Status status = strategy_->Process(cc); !status.ok()) {
-    // e.g. partial period at the end of the stream.
+    return status;  // Avoid MP_RETURN_IF_ERROR macro for external release.
-    if (packet_reservoir_->IsEnabled() &&
-        (first_timestamp_ == Timestamp::Unset() ||
-         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
-      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
-      packet_reservoir_->AddSample(curr_packet);
-    }
-    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
-  } else {
-    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
   }
+
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
-  return absl::OkStatus();
-}
 
-void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
-  next_output_timestamp_min_ = first_timestamp_;
-  if (jitter_with_reflection_) {
-    next_output_timestamp_ =
-        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
-    return;
-  }
-  next_output_timestamp_ =
-      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
-}
-
-void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
-  packet_reservoir_->Clear();
-  if (jitter_with_reflection_) {
-    next_output_timestamp_min_ += frame_time_usec_;
-    Timestamp next_output_timestamp_max_ =
-        next_output_timestamp_min_ + frame_time_usec_;
-
-    next_output_timestamp_ += frame_time_usec_ +
-                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
-                              jitter_usec_;
-    next_output_timestamp_ = Timestamp(ReflectBetween(
-        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
-        next_output_timestamp_max_.Value()));
-    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
-    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
-    return;
-  }
-  packet_reservoir_->Disable();
-  next_output_timestamp_ +=
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_NE(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    first_timestamp_ = cc->InputTimestamp();
-    InitializeNextOutputTimestampWithJitter();
-    if (first_timestamp_ == next_output_timestamp_) {
-      OutputWithinLimits(
-          cc,
-          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
-      UpdateNextOutputTimestampWithJitter();
-    }
-    return absl::OkStatus();
-  }
-
-  if (frame_time_usec_ <
-      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
-    LOG_FIRST_N(WARNING, 2)
-        << "Adding jitter is not very useful when upsampling.";
-  }
-
-  while (true) {
-    const int64 last_diff =
-        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-    RET_CHECK_GT(last_diff, 0);
-    const int64 curr_diff =
-        (next_output_timestamp_ - cc->InputTimestamp()).Value();
-    if (curr_diff > 0) {
-      break;
-    }
-    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
-                                ? last_packet_
-                                : cc->Inputs().Get(input_data_id_).Value())
-                               .At(next_output_timestamp_));
-    UpdateNextOutputTimestampWithJitter();
-    // From now on every time a packet is emitted the timestamp of the next
-    // packet becomes known; that timestamp is stored in next_output_timestamp_.
-    // The only exception to this rule is the packet emitted from Close() which
-    // can only happen when jitter_with_reflection is enabled but in this case
-    // next_output_timestamp_min_ is a non-decreasing lower bound of any
-    // subsequent packet.
-    const Timestamp timestamp_bound = jitter_with_reflection_
-                                          ? next_output_timestamp_min_
-                                          : next_output_timestamp_;
-    cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
-  }
-  return absl::OkStatus();
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_EQ(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    // This is the first packet, initialize the first_timestamp_.
-    if (base_timestamp_ == Timestamp::Unset()) {
-      // Initialize first_timestamp_ with exactly the first packet timestamp.
-      first_timestamp_ = cc->InputTimestamp();
-    } else {
-      // Initialize first_timestamp_ with the first packet timestamp
-      // aligned to the base_timestamp_.
-      int64 first_index = MathUtil::SafeRound<int64, double>(
-          (cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
-      first_timestamp_ =
-          base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
-    }
-    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
-      cc->Outputs()
-          .Tag("VIDEO_HEADER")
-          .Add(new VideoHeader(video_header_), Timestamp::PreStream());
-    }
-  }
-  const Timestamp received_timestamp = cc->InputTimestamp();
-  const int64 received_timestamp_idx =
-      TimestampToPeriodIndex(received_timestamp);
-  // Only consider the received packet if it belongs to the current period
-  // (== period_count_) or to a newer one (> period_count_).
-  if (received_timestamp_idx >= period_count_) {
-    // Fill the empty periods until we are in the same index as the received
-    // packet.
-    while (received_timestamp_idx > period_count_) {
-      OutputWithinLimits(
-          cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
-      ++period_count_;
-    }
-    // Now, if the received packet has a timestamp larger than the middle of
-    // the current period, we can send a packet without waiting. We send the
-    // one closer to the middle.
-    Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
-    if (received_timestamp >= target_timestamp) {
-      bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
-      bool send_current =
-          !have_last_packet || (received_timestamp - target_timestamp <=
-                                target_timestamp - last_packet_.Timestamp());
-      if (send_current) {
-        OutputWithinLimits(
-            cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
-      } else {
-        OutputWithinLimits(cc, last_packet_.At(target_timestamp));
-      }
-      ++period_count_;
-    }
-    // TODO: Add a mechanism to the framework to allow these packets
-    // to be output earlier (without waiting for a much later packet to
-    // arrive)
-
-    // Update the bound for the next packet.
-    cc->Outputs()
-        .Get(output_data_id_)
-        .SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
-  }
   return absl::OkStatus();
 }
 
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
   if (!cc->GraphStatus().ok()) {
     return absl::OkStatus();
   }
-  // Emit the last packet received if we have at least one packet, but
+
-  // haven't sent anything for its period.
+  return strategy_->Close(cc);
-  if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
+}
-      TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
+
-    OutputWithinLimits(cc,
+std::unique_ptr<PacketResamplerStrategy>
-                       last_packet_.At(PeriodIndexToTimestamp(period_count_)));
+PacketResamplerCalculator::GetSamplingStrategy(
+    const PacketResamplerCalculatorOptions& options) {
+  if (options.reproducible_sampling()) {
+    if (!options.jitter_with_reflection()) {
+      LOG(WARNING)
+          << "reproducible_sampling enabled w/ jitter_with_reflection "
+             "disabled. "
+          << "reproducible_sampling always uses jitter with reflection, "
+          << "Ignoring jitter_with_reflection setting.";
+    }
+    return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
   }
-  if (!packet_reservoir_->IsEmpty()) {
+
-    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  if (options.jitter() == 0) {
+    return absl::make_unique<NoJitterStrategy>(this);
   }
-  return absl::OkStatus();
+
+  if (options.jitter_with_reflection()) {
+    return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
+  }
+
+  // With jitter and no reflection.
+  return absl::make_unique<JitterWithoutReflectionStrategy>(this);
 }
 
 Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
   }
 }
 
+absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    LOG(INFO) << "Emitting pack from reservoir.";
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (packet_reservoir_->IsEnabled() &&
+      (first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (first_timestamp_ == Timestamp::Unset()) {
+    first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestampWithJitter();
+    if (first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
+    // From now on every time a packet is emitted the timestamp of the next
+    // packet becomes known; that timestamp is stored in next_output_timestamp_.
+    // The only exception to this rule is the packet emitted from Close() which
+    // can only happen when jitter_with_reflection is enabled but in this case
+    // next_output_timestamp_min_ is a non-decreasing lower bound of any
+    // subsequent packet.
+    const Timestamp timestamp_bound = next_output_timestamp_min_;
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(timestamp_bound);
+  }
+  return absl::OkStatus();
+}
+
+void LegacyJitterWithReflectionStrategy::
+    InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
+  next_output_timestamp_ =
+      first_timestamp_ +
+      random_->UnbiasedUniform64(calculator_->frame_time_usec_);
+}
+
+void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  next_output_timestamp_min_ += calculator_->frame_time_usec_;
+  Timestamp next_output_timestamp_max_ =
+      next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+  next_output_timestamp_ +=
+      calculator_->frame_time_usec_ +
+      random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
+      calculator_->jitter_usec_;
+  next_output_timestamp_ = Timestamp(ReflectBetween(
+      next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+      next_output_timestamp_max_.Value()));
+  CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+  CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+}
+
+absl::Status ReproducibleJitterWithReflectionStrategy::Open(
+    CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Close(
+    CalculatorContext* cc) {
+  // If last packet is non-empty and a packet hasn't been emitted for this
+  // period, emit the last packet.
+  if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(next_output_timestamp_));
+  }
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+
+  if (calculator_->last_packet_.IsEmpty()) {
+    // last_packet is empty, this is the first packet of the stream.
+
+    InitializeNextOutputTimestamp(current_packet.Timestamp());
+
+    // If next_output_timestamp_ happens to fall before current_packet, emit
+    // current packet.  Only a single packet can be emitted at the beginning
+    // of the stream.
+    if (next_output_timestamp_ < current_packet.Timestamp()) {
+      calculator_->OutputWithinLimits(
+          cc, current_packet.At(next_output_timestamp_));
+      packet_emitted_this_period_ = true;
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Last packet is set, so we are mid-stream.
+  if (calculator_->frame_time_usec_ <
+      (current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
+          .Value()) {
+    // Note, if the stream is upsampling, this could lead to the same packet
+    // being emitted twice.  Upsampling and jitter doesn't make much sense
+    // but does technically work.
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  // Since we may be upsampling, we need to iteratively advance the
+  // next_output_timestamp_ one period at a time until it reaches the period
+  // current_packet is in.  During this process, last_packet and/or
+  // current_packet may be repeatly emitted.
+
+  UpdateNextOutputTimestamp(current_packet.Timestamp());
+
+  while (!packet_emitted_this_period_ &&
+         next_output_timestamp_ <= current_packet.Timestamp()) {
+    // last_packet < next_output_timestamp_ <= current_packet,
+    // so emit the closest packet.
+    Packet packet_to_emit =
+        current_packet.Timestamp() - next_output_timestamp_ <
+                next_output_timestamp_ - calculator_->last_packet_.Timestamp()
+            ? current_packet
+            : calculator_->last_packet_;
+    calculator_->OutputWithinLimits(cc,
+                                    packet_to_emit.At(next_output_timestamp_));
+
+    packet_emitted_this_period_ = true;
+
+    // If we are upsampling, packet_emitted_this_period_ can be reset by
+    // the following UpdateNext and the loop will iterate.
+    UpdateNextOutputTimestamp(current_packet.Timestamp());
+  }
+
+  // Set the bounds on the output stream.  Note, if we emitted a packet
+  // above, it will already be set at next_output_timestamp_ + 1, in which
+  // case we have to skip setting it.
+  if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
+      next_output_timestamp_) {
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (next_output_timestamp_min_ != Timestamp::Unset()) {
+    return;
+  }
+
+  next_output_timestamp_min_ = Timestamp(0);
+  next_output_timestamp_ =
+      Timestamp(GetNextRandom(calculator_->frame_time_usec_));
+
+  // While the current timestamp is ahead of the max (i.e. min + frame_time),
+  // fast-forward.
+  while (current_timestamp >=
+         next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    packet_emitted_this_period_ = true;  // Force update...
+    UpdateNextOutputTimestamp(current_timestamp);
+  }
+}
+
+void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (packet_emitted_this_period_ &&
+      current_timestamp >=
+          next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    next_output_timestamp_min_ += calculator_->frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+    next_output_timestamp_ += calculator_->frame_time_usec_ +
+                              GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
+                              calculator_->jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+
+    packet_emitted_this_period_ = false;
+  }
+}
+
+absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  // Packet reservior is used to make sure there's an output for every period,
+  // e.g. partial period at the end of the stream.
+  if (packet_reservoir_->IsEnabled() &&
+      (calculator_->first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    calculator_->first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestamp();
+    if (calculator_->first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestamp();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestamp();
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
+  next_output_timestamp_min_ = calculator_->first_timestamp_;
+  next_output_timestamp_ = calculator_->first_timestamp_ +
+                           calculator_->frame_time_usec_ * random_->RandFloat();
+}
+
+void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
+  packet_reservoir_->Clear();
+  packet_reservoir_->Disable();
+  next_output_timestamp_ += calculator_->frame_time_usec_ *
+                            ((1.0 - calculator_->jitter_) +
+                             2.0 * calculator_->jitter_ * random_->RandFloat());
+}
+
+absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+  base_timestamp_ = resampler_options.has_base_timestamp()
+                        ? Timestamp(resampler_options.base_timestamp())
+                        : Timestamp::Unset();
+
+  period_count_ = 0;
+
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
+  // Emit the last packet received if we have at least one packet, but
+  // haven't sent anything for its period.
+  if (calculator_->first_timestamp_ != Timestamp::Unset() &&
+      calculator_->flush_last_packet_ &&
+      calculator_->TimestampToPeriodIndex(
+          calculator_->last_packet_.Timestamp()) == period_count_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(
+                calculator_->PeriodIndexToTimestamp(period_count_)));
+  }
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    // This is the first packet, initialize the first_timestamp_.
+    if (base_timestamp_ == Timestamp::Unset()) {
+      // Initialize first_timestamp_ with exactly the first packet timestamp.
+      calculator_->first_timestamp_ = cc->InputTimestamp();
+    } else {
+      // Initialize first_timestamp_ with the first packet timestamp
+      // aligned to the base_timestamp_.
+      int64 first_index = MathUtil::SafeRound<int64, double>(
+          (cc->InputTimestamp() - base_timestamp_).Seconds() *
+          calculator_->frame_rate_);
+      calculator_->first_timestamp_ =
+          base_timestamp_ +
+          TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
+    }
+    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
+      cc->Outputs()
+          .Tag("VIDEO_HEADER")
+          .Add(new VideoHeader(calculator_->video_header_),
+               Timestamp::PreStream());
+    }
+  }
+  const Timestamp received_timestamp = cc->InputTimestamp();
+  const int64 received_timestamp_idx =
+      calculator_->TimestampToPeriodIndex(received_timestamp);
+  // Only consider the received packet if it belongs to the current period
+  // (== period_count_) or to a newer one (> period_count_).
+  if (received_timestamp_idx >= period_count_) {
+    // Fill the empty periods until we are in the same index as the received
+    // packet.
+    while (received_timestamp_idx > period_count_) {
+      calculator_->OutputWithinLimits(
+          cc, calculator_->last_packet_.At(
+                  calculator_->PeriodIndexToTimestamp(period_count_)));
+      ++period_count_;
+    }
+    // Now, if the received packet has a timestamp larger than the middle of
+    // the current period, we can send a packet without waiting. We send the
+    // one closer to the middle.
+    Timestamp target_timestamp =
+        calculator_->PeriodIndexToTimestamp(period_count_);
+    if (received_timestamp >= target_timestamp) {
+      bool have_last_packet =
+          (calculator_->last_packet_.Timestamp() != Timestamp::Unset());
+      bool send_current =
+          !have_last_packet ||
+          (received_timestamp - target_timestamp <=
+           target_timestamp - calculator_->last_packet_.Timestamp());
+      if (send_current) {
+        calculator_->OutputWithinLimits(cc,
+                                        cc->Inputs()
+                                            .Get(calculator_->input_data_id_)
+                                            .Value()
+                                            .At(target_timestamp));
+      } else {
+        calculator_->OutputWithinLimits(
+            cc, calculator_->last_packet_.At(target_timestamp));
+      }
+      ++period_count_;
+    }
+    // TODO: Add a mechanism to the framework to allow these packets
+    // to be output earlier (without waiting for a much later packet to
+    // arrive)
+
+    // Update the bound for the next packet.
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(
+            calculator_->PeriodIndexToTimestamp(period_count_));
+  }
+  return absl::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -55,7 +55,7 @@ class PacketReservoir {
 //     correspond to timestamp t.
 //   - The next packet is chosen randomly (uniform distribution) among frames
 //     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
-//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//     - if jitter_with_reflection is true, the timestamp will be reflected
 //       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
 //       so that its marginal distribution is uniform within this interval.
 //       In the formula, t_0 is the timestamp of the first sampled
@@ -66,6 +66,17 @@ class PacketReservoir {
 //     the resampling.  For Cloud ML Video Intelligence API, the hash of the
 //     input video should serve this purpose.  For YouTube, either video ID or
 //     content hex ID of the input video should do.
+//   - If reproducible_samping is true, care is taken to allow reproducible
+//     "mid-stream" sampling.  The calculator can be executed on a stream that
+//     doesn't start at the first period.  For instance, if the calculator
+//     is run on a 10 second stream it will produce the same set of samples
+//     as two runs of the calculator, the first with 3 seconds of input starting
+//     at time 0 and the second with 7 seconds of input starting at time +3s.
+//     - In order to guarantee the exact same samples, 1) the inputs must be
+//       aligned with the sampling period.  For instance, if the sampling rate
+//       is 2 frames per second, streams should be aligned on 0.5 second
+//       boundaries, and 2) the stream must include at least one extra packet
+//       before and after the second aligned sampling period.
 //
 // If jitter_ is not specified:
 //   - The first packet defines the first_timestamp of the output stream,
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
   absl::Status Process(CalculatorContext* cc) override;
 
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  absl::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  absl::Status ProcessWithoutJitter(CalculatorContext* cc);
-
   // Given the current count of periods that have passed, this returns
   // the next valid timestamp of the middle point of the next period:
   //    if count is 0, it returns the first_timestamp_.
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Outputs a packet if it is in range (start_time_, end_time_).
   void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
 
+ protected:
+  // Returns Sampling Strategy to use.
+  //
+  // Virtual to allow injection of testing strategies.
+  virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& options);
+
+ private:
+  std::unique_ptr<class PacketResamplerStrategy> strategy_;
+
   // The timestamp of the first packet received.
   Timestamp first_timestamp_;
 
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Inverse of frame_rate_.
   int64 frame_time_usec_;
 
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
   VideoHeader video_header_;
   // The "DATA" input stream.
   CollectionItemId input_data_id_;
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
   CollectionItemId output_data_id_;
 
   // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  // than the final stream timestamp.
   bool flush_last_packet_;
 
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
   double jitter_ = 0.0;
-  bool jitter_with_reflection_;
-  int64 jitter_usec_;
-  Timestamp next_output_timestamp_;
-  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
-  // kept within the interval
-  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
-  Timestamp next_output_timestamp_min_;
 
-  // If specified, output timestamps are aligned with base_timestamp.
+  int64 jitter_usec_;
-  // Otherwise, they are aligned with the first input timestamp.
+
-  Timestamp base_timestamp_;
+  // The last packet that was received.
+  Packet last_packet_;
 
   // If specified, only outputs at/after start_time are included.
   Timestamp start_time_;
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
 
   // If set, the output timestamps nearest to start_time and end_time
   // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.W
+  // between start_time and end_time.
   bool round_limits_;
 
+  // Allow strategies access to all internal calculator state.
+  //
+  // The calculator and strategies are intimiately tied together so this should
+  // not break encapsulation.
+  friend class LegacyJitterWithReflectionStrategy;
+  friend class ReproducibleJitterWithReflectionStrategy;
+  friend class JitterWithoutReflectionStrategy;
+  friend class NoJitterStrategy;
+};
+
+// Abstract class encapsulating sampling stategy.
+//
+// These are used solely by PacketResamplerCalculator, but are exposed here
+// to facilitate tests.
+class PacketResamplerStrategy {
+ public:
+  PacketResamplerStrategy(PacketResamplerCalculator* calculator)
+      : calculator_(calculator) {}
+  virtual ~PacketResamplerStrategy() = default;
+
+  // Delegate for CalculatorBase::Open.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Open(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Close.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Close(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Process.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Process(CalculatorContext* cc) = 0;
+
+ protected:
+  // Calculator running strategy.
+  PacketResamplerCalculator* calculator_;
+};
+
+// Strategy that applies Jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+//
+// This applies the legacy jitter with reflection which doesn't allow
+// for reproducibility of sampling when starting mid-stream.  This is maintained
+// for backward compatibility.
+class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void InitializeNextOutputTimestampWithJitter();
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
   // packet reservior used for sampling random packet out of partial
   // period when jitter is enabled
   std::unique_ptr<PacketReservoir> packet_reservoir_;
+
   // random number generator used in packet_reservior_.
   std::unique_ptr<RandomBase> packet_reservoir_random_;
 };
 
+// Strategy that applies reproducible jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+class ReproducibleJitterWithReflectionStrategy
+    : public PacketResamplerStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategy(
+      PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ protected:
+  // Returns next random in range (0,n].
+  //
+  // Exposed as virtual function for testing Jitter with reflection.
+  // This is the only way random_ is accessed.
+  virtual uint64 GetNextRandom(uint64 n) {
+    return random_->UnbiasedUniform64(n);
+  }
+
+ private:
+  // Initializes Jitter with reflection.
+  //
+  // This will fast-forward to the period containing current_timestamp.
+  // next_output_timestamp_ is guarnateed to be current_timestamp's period
+  // and packet_emitted_this_period_ will be set to false.
+  void InitializeNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Potentially advances next_output_timestamp_ a single period.
+  //
+  // next_output_timestamp_ will only be advanced if packet_emitted_this_period_
+  // is false.  next_output_timestamp_ will never be advanced beyond
+  // current_timestamp's period.
+  //
+  // However, next_output_timestamp_ could fall before current_timestamp's
+  // period since only a single period can be advanced at a time.
+  void UpdateNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // Indicates packet was emitted for current period (i.e. the period
+  // next_output_timestamp_ falls in.
+  bool packet_emitted_this_period_ = false;
+};
+
+// Strategy that applies Jitter without reflection based sampling.
+//
+// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
+// not enabled.
+class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestamp();
+
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestamp();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // packet reservior used for sampling random packet out of partial period.
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+// Strategy that applies sampling without any jitter.
+//
+// Used by PacketResamplerCalculator when jitter is not enabled.
+class NoJitterStrategy : public PacketResamplerStrategy {
+ public:
+  NoJitterStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Number of periods that have passed (= #packets sent to the output).
+  int64 period_count_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+};
+
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+
+  // Enables reflection when applying jitter.
+  //
+  // This option is ignored when reproducible_sampling is true, in which case
+  // reflection will be used.
+  //
+  // New use cases should use reproducible_sampling = true, as
+  // jitter_with_reflection is deprecated and will be removed at some point.
   optional bool jitter_with_reflection = 9 [default = false];
 
+  // If set, enabled reproducible sampling, allowing frames to be sampled
+  // without regards to where the stream starts.  See
+  // packet_resampler_calculator.h for details.
+  //
+  // This enables reflection (ignoring jitter_with_reflection setting).
+  optional bool reproducible_sampling = 10 [default = false];
+
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.
   //

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -30,6 +30,7 @@
 
 namespace mediapipe {
 
+using ::testing::ElementsAre;
 namespace {
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
   static int static_count_;
 };
 
+// Matcher for Packets with uint64 payload, comparing arg packet's
+// timestamp and uint64 payload.
+MATCHER_P2(PacketAtTimestamp, payload, timestamp,
+           absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
+                        payload, " @ time ", timestamp)) {
+  if (timestamp != arg.Timestamp().Value()) {
+    *result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
+    return false;
+  }
+  int64 actual_payload = arg.template Get<int64>();
+  if (actual_payload != payload) {
+    *result_listener << "with incorrect payload = " << actual_payload;
+    return false;
+  }
+  return true;
+}
+
+// JitterWithReflectionStrategy child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleJitterWithReflectionStrategyForTesting
+    : public ReproducibleJitterWithReflectionStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategyForTesting(
+      PacketResamplerCalculator* calculator)
+      : ReproducibleJitterWithReflectionStrategy(calculator) {}
+
+  // Statically accessed random sequence to use for jitter with reflection.
+  //
+  // An EXPECT will fail if sequence is less than the number requested during
+  // processing.
+  static std::vector<uint64> random_sequence;
+
+ protected:
+  virtual uint64 GetNextRandom(uint64 n) {
+    EXPECT_LT(sequence_index_, random_sequence.size());
+    return random_sequence[sequence_index_++] % n;
+  }
+
+ private:
+  int32 sequence_index_ = 0;
+};
+std::vector<uint64>
+    ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
+
+// PacketResamplerCalculator child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleResamplerCalculatorForTesting
+    : public PacketResamplerCalculator {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    return PacketResamplerCalculator::GetContract(cc);
+  }
+
+ protected:
+  std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& Options) {
+    return absl::make_unique<
+        ReproducibleJitterWithReflectionStrategyForTesting>(this);
+  }
+};
+
+REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
+
 int SimpleRunner::static_count_ = 0;
 
 TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
@@ -380,7 +452,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
 }
 
 TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "PacketResamplerCalculator"
     input_stream: "DATA:in_data"
     input_stream: "VIDEO_HEADER:in_video_header"
@@ -389,7 +461,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
     options {
       [mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 }
     }
-  )"));
+  )pb"));
 
   for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
     runner.MutableInputs()->Tag("DATA").packets.push_back(
@@ -633,7 +705,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
 
 TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "PacketResamplerCalculator"
         input_side_packet: "OPTIONS:options"
         input_stream: "input"
@@ -643,16 +715,16 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
             frame_rate: 60
             base_timestamp: 0
           }
-        })");
+        })pb");
 
   {
     SimpleRunner runner(node_config);
     auto options =
         new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(
-            R"(
+            R"pb(
               [mediapipe.PacketResamplerCalculatorOptions.ext] {
                 frame_rate: 30
-              })"));
+              })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
     MP_ASSERT_OK(runner.Run());
@@ -662,12 +734,12 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
     SimpleRunner runner(node_config);
 
     auto options =
-        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"(
+        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"pb(
           merge_fields: false
           [mediapipe.PacketResamplerCalculatorOptions.ext] {
             frame_rate: 30
             base_timestamp: 0
-          })"));
+          })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -69,7 +69,7 @@ MATCHER_P2(PairPacket, timestamp, pair, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> in_prev;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -93,7 +93,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'previous2'
           output_stream: 'pair'
         }
-      )");
+      )pb");
   tool::AddVectorSink("pair", &graph_config_, &in_prev);
 
   CalculatorGraph graph_;
@@ -169,7 +169,7 @@ REGISTER_CALCULATOR(PacketOnCloseCalculator);
 TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -192,7 +192,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -231,7 +231,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -253,7 +253,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -278,7 +278,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -300,7 +300,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -331,7 +331,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
 TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -354,7 +354,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -386,7 +386,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           input_stream: 'force_main_empty'
           input_stream: 'force_loop_empty'
@@ -424,7 +424,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
@@ -724,7 +724,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           # Drops "loop" when set to "true", delaying output of prev_loop, hence
           # delaying output of the graph.
@@ -755,7 +755,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -53,7 +53,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -63,7 +63,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -80,7 +80,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -90,7 +90,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -107,7 +107,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -117,7 +117,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
             min_quantized_value: -1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -133,7 +133,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -143,7 +143,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
   runner.MutableInputs()
@@ -171,7 +171,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -181,7 +181,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {-65.0f, 65.0f};
   runner.MutableInputs()

mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc

@@ -70,13 +70,13 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
 constexpr int kNumImageFrames = 5;
 constexpr int kNumFinished = 3;
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "RealTimeFlowLimiterCalculator"
     input_stream: "raw_frames"
     input_stream: "FINISHED:finished"
     input_stream_info: { tag_index: "FINISHED" back_edge: true }
     output_stream: "gated_frames"
-  )");
+  )pb");
 }
 
 // Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
@@ -219,7 +219,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The two LambdaCalculators are used to keep certain packet sets in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       input_stream: 'in_2'
       node {
@@ -256,7 +256,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
         output_stream: 'out_1'
         output_stream: 'out_2'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -344,7 +344,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
   std::vector<Packet> a_passed;
   std::vector<Packet> b_passed;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_a'
         input_stream: 'in_b'
         input_stream: 'finished'
@@ -360,7 +360,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
           output_stream: 'in_b_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
   tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
@@ -442,7 +442,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
 TEST(RealTimeFlowLimiterCalculator, CanConsume) {
   std::vector<Packet> in_sampled_packets_;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         input_stream: 'finished'
         node {
@@ -455,7 +455,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
           output_stream: 'in_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
   tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);

mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc

@@ -36,7 +36,7 @@ using testing::HasSubstr;
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -45,7 +45,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -58,7 +58,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -67,7 +67,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -79,7 +79,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -88,7 +88,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
               input_side_packet: "side_packet"
               output_stream: "DOES_NOT_EXIST:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -102,7 +102,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -113,7 +113,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
               output_stream: "AT_TICK:packet0"
               output_stream: "AT_PRE_STREAM:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -127,7 +127,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -136,7 +136,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
               output_stream: "AT_PRESTREAM:0:packet0"
               output_stream: "AT_PRESTREAM:1:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -149,7 +149,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -158,7 +158,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
               input_side_packet: "side_packet1"
               output_stream: "AT_PRESTREAM:packet0"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -209,7 +209,7 @@ TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
 TEST(SidePacketToStreamCalculator, AtTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -219,7 +219,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -251,7 +251,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
 TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -265,7 +265,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
               output_stream: "AT_TICK:0:packet0"
               output_stream: "AT_TICK:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;
@@ -305,7 +305,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
 TEST(SidePacketToStreamCalculator, AtTimestamp) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -315,7 +315,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -337,7 +337,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
 TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -350,7 +350,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
               output_stream: "AT_TIMESTAMP:0:packet0"
               output_stream: "AT_TIMESTAMP:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -122,7 +122,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -138,7 +138,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -171,7 +171,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -183,7 +183,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -196,7 +196,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -209,7 +209,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -223,7 +223,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -238,7 +238,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -252,7 +252,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -266,7 +266,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -281,7 +281,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -298,7 +298,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -334,7 +334,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -349,7 +349,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -377,7 +377,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -394,7 +394,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -163,7 +163,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -179,7 +179,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -214,7 +214,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -226,7 +226,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -240,7 +240,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -253,7 +253,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -269,7 +269,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -284,7 +284,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -299,7 +299,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -313,7 +313,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -330,7 +330,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -347,7 +347,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -385,7 +385,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -400,7 +400,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -428,7 +428,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -445,7 +445,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -511,7 +511,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -524,7 +524,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -536,7 +536,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -552,7 +552,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -592,7 +592,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -609,7 +609,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -646,7 +646,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -661,7 +661,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -41,7 +41,7 @@ constexpr char kWidthTag[] = "WIDTH";
 TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -54,7 +54,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -79,7 +79,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -94,7 +94,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -119,7 +119,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
 TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "WIDTH:crop_width"
@@ -136,7 +136,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -168,7 +168,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "RECT:rect"
@@ -184,7 +184,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -196,9 +196,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
       calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
   auto& inputs = cc->Inputs();
   mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
-      R"(
+      R"pb(
         width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
-      )");
+      )pb");
   inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
   RectSpec expectRect = {
       .width = 1,

mediapipe/calculators/image/image_file_properties_calculator_test.cc

@@ -50,11 +50,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_stream: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -79,11 +79,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_side_packet: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Index(0) =
@@ -108,11 +108,11 @@ TEST(ImageFilePropertiesCalculatorTest,
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc

@@ -36,11 +36,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
   Packet input_packet = MakePacket<std::string>(contents);
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));
@@ -79,11 +79,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
       reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));

mediapipe/calculators/tensor/BUILD

@@ -561,13 +561,13 @@ cc_test(
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -15,6 +15,7 @@
 #include <cmath>
 #include <vector>
 
+#include "absl/flags/flag.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
@@ -28,7 +29,6 @@
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"

mediapipe/calculators/tensor/inference_calculator.cc

@@ -41,7 +41,6 @@ class InferenceCalculatorSelectorImpl
         (options.has_delegate() && options.delegate().has_gpu());
     if (should_use_gpu) {
       impls.emplace_back("Metal");
-      impls.emplace_back("MlDrift");
       impls.emplace_back("Gl");
     }
     impls.emplace_back("Cpu");

mediapipe/calculators/tensor/inference_calculator.h

@@ -118,10 +118,6 @@ struct InferenceCalculatorGl : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
 };
 
-struct InferenceCalculatorMlDrift : public InferenceCalculator {
-  static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
-};
-
 struct InferenceCalculatorMetal : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
 };

mediapipe/calculators/tensor/inference_calculator.proto

@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Choose any of available APIs to force running inference using it.
-      enum API {
+      enum Api {
         ANY = 0;
         OPENGL = 1;
         OPENCL = 2;
       }
-      optional API api = 4 [default = ANY];
+      optional Api api = 4 [default = ANY];
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Set to true to use 16-bit float precision. If max precision is needed,

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -136,7 +136,7 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);

mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc

@@ -57,11 +57,7 @@ const std::vector<Param>& GetParams() {
     // Metal is not available on the iOS simulator.
     p.push_back({"Metal", "Metal"});
     p.back().delegate.mutable_gpu();
-#endif  // TARGET_IPHONE_SIMULATOR
+#endif                // TARGET_IPHONE_SIMULATOR
-#if __EMSCRIPTEN__
-    p.push_back({"MlDrift", "MlDrift"});
-    p.back().delegate.mutable_gpu();
-#endif                // __EMSCRIPTEN__
 #if __ANDROID__ && 0  // Disabled for now since emulator can't go GLESv3
     p.push_back({"Gl", "Gl"});
     p.back().delegate.mutable_gpu();
@@ -78,18 +74,6 @@ const std::vector<Param>& GetParams() {
 
 class InferenceCalculatorTest : public testing::TestWithParam<Param> {
  protected:
-#if __EMSCRIPTEN__
-  // TODO: fix Tensor locking.
-  // The MlDrift backend currently fails in debug mode without this,
-  // because of Tensor locking issues. I am adding this temporarily since
-  // the calculator is already being used and it's better to have test
-  // coverage for it. Also, the issue doesn't apply to our Emscripten
-  // build in practice since it's single-threaded.
-  void SetUp(void) override {
-    absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kIgnore);
-  }
-#endif  // __EMSCRIPTEN__
-
   void SetDelegateForParam(mediapipe::CalculatorGraphConfig_Node* node) {
     *node->mutable_options()
          ->MutableExtension(mediapipe::InferenceCalculatorOptions::ext)

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -63,7 +63,7 @@ class InferenceCalculatorGlImpl
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
   bool allow_precision_loss_ = false;
-  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::API
+  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
       tflite_gpu_runner_api_;
 #endif  // MEDIAPIPE_TFLITE_GL_INFERENCE
 
@@ -244,7 +244,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   // Create runner
   tflite::gpu::InferenceOptions options;
@@ -294,7 +294,7 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -200,7 +200,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
   const auto& model = *model_packet_.Get();
   tflite::ops::builtin::BuiltinOpResolver op_resolver =
       kSideInCustomOpResolver(cc).GetOr(
-          tflite::ops::builtin::BuiltinOpResolver());
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates());
 
   tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
   RET_CHECK(interpreter_);

mediapipe/calculators/tensor/tensor_converter_calculator_test.cc

@@ -84,7 +84,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TensorConverterCalculator"
@@ -96,7 +96,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -146,7 +146,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TensorConverterCalculator"
@@ -158,7 +158,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -205,7 +205,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
   CalculatorGraph graph;
   // Run the calculator and verify that one output is generated.
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_image"
         node {
           calculator: "TensorConverterCalculator"
@@ -220,7 +220,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
             }
           }
         }
-      )");
+      )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor", &graph_config, &output_packets);
 

mediapipe/calculators/tensor/tensors_to_classification_calculator_test.cc

@@ -56,14 +56,14 @@ class TensorsToClassificationCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
     options {
       [mediapipe.TensorsToClassificationCalculatorOptions.ext] {}
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -85,7 +85,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) {
 }
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -94,7 +94,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
         label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt"
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -117,7 +117,7 @@ TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) {
 
 TEST_F(TensorsToClassificationCalculatorTest,
        CorrectOutputWithLabelMinScoreThreshold) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -126,7 +126,7 @@ TEST_F(TensorsToClassificationCalculatorTest,
         min_score_threshold: 0.6
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -144,14 +144,14 @@ TEST_F(TensorsToClassificationCalculatorTest,
 }
 
 TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
     options {
       [mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());

mediapipe/calculators/tensor/tensors_to_floats_calculator_test.cc

@@ -57,11 +57,11 @@ class TensorsToFloatsCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOAT:float"
-  )"));
+  )pb"));
 
   const float single_value = 0.5;
   BuildGraph(&runner, {single_value});
@@ -76,11 +76,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValue) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
-  )"));
+  )pb"));
 
   const float single_value = 0.5;
   BuildGraph(&runner, {single_value});
@@ -95,11 +95,11 @@ TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
-  )"));
+  )pb"));
 
   const std::vector<float> input_values = {0.f, 0.5f, 1.0f};
   BuildGraph(&runner, input_values);
@@ -116,14 +116,14 @@ TEST_F(TensorsToFloatsCalculatorTest, FloatVector) {
 }
 
 TEST_F(TensorsToFloatsCalculatorTest, FloatVectorWithSigmoid) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TensorsToFloatsCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "FLOATS:floats"
     options {
       [mediapipe.TensorsToFloatsCalculatorOptions.ext] { activation: SIGMOID }
     }
-  )"));
+  )pb"));
 
   const std::vector<float> input_values = {-1.f, 0.f, 1.0f};
   const std::vector<float> expected_output_with_sigmoid = {0.269f, 0.5f,

mediapipe/calculators/tensorflow/BUILD

@@ -892,13 +892,13 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
@@ -923,13 +923,13 @@ cc_test(
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
@@ -954,11 +954,11 @@ cc_test(
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
@@ -981,11 +981,11 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
@@ -1144,8 +1144,8 @@ cc_test(
         ":tensorflow_inference_calculator",
         ":tensorflow_session_from_frozen_graph_generator",
         ":tensorflow_session_from_frozen_graph_generator_cc_proto",
+        "@com_google_absl//absl/flags:flag",
         "//mediapipe/framework/deps:file_path",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",

mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc

@@ -68,31 +68,31 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
 
   void CreateNodeConfig(CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "NUM_DETECTIONS:num_detections"
       input_stream: "BOXES:boxes"
       input_stream: "SCORES:scores"
       input_stream: "CLASSES:classes"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void CreateNodeConfigRawTensors(
       CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "BOXES:raw_detection_boxes"
       input_stream: "SCORES:raw_detection_scores"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void CreateNodeConfigWithKeypoints(
       CalculatorGraphConfig::Node* node_config) const {
     ASSERT_NE(nullptr, node_config);
-    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    *node_config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
       calculator: "ObjectDetectionTensorsToDetectionsCalculator"
       input_stream: "NUM_DETECTIONS:num_detections"
       input_stream: "BOXES:boxes"
@@ -100,7 +100,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
       input_stream: "CLASSES:classes"
       input_stream: "KEYPOINTS:keypoints"
       output_stream: "DETECTIONS:detections"
-    )");
+    )pb");
   }
 
   void SetUpCalculatorRunner() {
@@ -177,7 +177,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
     InsertExtraSingltonDim(&input_scores_);
     InsertExtraSingltonDim(&input_classes_);
     CalculatorGraphConfig::Node node_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
           calculator: "ObjectDetectionTensorsToDetectionsCalculator"
           input_stream: "NUM_DETECTIONS:num_detections"
           input_stream: "BOXES:boxes"
@@ -188,7 +188,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
             [mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions
                  .ext]: { tensor_dim_to_squeeze: 0 }
           }
-        )");
+        )pb");
     runner_ = absl::make_unique<CalculatorRunner>(node_config);
     runner_->MutableInputs()
         ->Tag(kNumDetections)

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -267,6 +267,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
     }
   }
 
+  absl::Status VerifySize() {
+    const int64 MAX_PROTO_BYTES = 1073741823;
+    std::string id = mpms::HasExampleId(*sequence_)
+                         ? mpms::GetExampleId(*sequence_)
+                         : "example";
+    RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES)
+        << "sequence '" << id
+        << "' would be too many bytes to serialize after adding features.";
+    return absl::OkStatus();
+  }
+
   absl::Status Close(CalculatorContext* cc) override {
     auto& options = cc->Options<PackMediaSequenceCalculatorOptions>();
     if (options.reconcile_metadata()) {
@@ -275,6 +286,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           options.reconcile_region_annotations(), sequence_.get()));
     }
 
+    if (options.skip_large_sequences()) {
+      RET_CHECK_OK(VerifySize());
+    }
     if (options.output_only_if_all_present()) {
       absl::Status status = VerifySequence();
       if (!status.ok()) {

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto

@@ -61,4 +61,8 @@ message PackMediaSequenceCalculatorOptions {
   // present, the previous images and timestamps will be removed before adding
   // the new images.
   optional bool replace_data_instead_of_append = 4 [default = true];
+
+  // If true, will return an error status if an output sequence would be too
+  // many bytes to serialize.
+  optional bool skip_large_sequences = 7 [default = true];
 }

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -889,5 +889,24 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
   MP_ASSERT_OK(runner_->Run());
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
+  SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+
+  // 1 billion floats should be > 1GB which can't be serialized. It should fail
+  // gracefully with this input.
+  int num_timesteps = 1000;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
+    runner_->MutableInputs()
+        ->Tag("FLOAT_FEATURE_TEST")
+        .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
+  }
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+  ASSERT_FALSE(runner_->Run().ok());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc

@@ -16,12 +16,12 @@
 #include <string>
 #include <vector>
 
+#include "absl/flags/flag.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
@@ -19,7 +20,6 @@
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
@@ -19,7 +20,6 @@
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/packet_generator.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
@@ -20,7 +21,6 @@
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
@@ -19,7 +20,6 @@
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/packet_generator.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/strings/match.h"
 #include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
 #include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h"

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto

@@ -56,11 +56,4 @@ message UnpackMediaSequenceCalculatorOptions {
   // the clip start and end times and outputs these for the
   // AudioDecoderCalculator to consume.
   optional AudioDecoderOptions base_audio_decoder_options = 9;
-
-  optional string keypoint_names = 10 [
-    default =
-        "NOSE,LEFT_EAR,RIGHT_EAR,LEFT_SHOULDER,RIGHT_SHOULDER,LEFT_FORE_PAW,RIGHT_FORE_PAW,LEFT_HIP,RIGHT_HIP,LEFT_HIND_PAW,RIGHT_HIND_PAW,ROOT_TAIL"
-  ];
-  // When the keypoint doesn't exists, output this default value.
-  optional float default_keypoint_location = 11 [default = -1.0];
 }

mediapipe/calculators/tflite/BUILD

@@ -147,11 +147,11 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 

mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc

@@ -12,11 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/object_detection/anchor.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -68,7 +68,7 @@ void CompareAnchors(const std::vector<Anchor>& anchors_0,
 }
 
 TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "SsdAnchorsCalculator"
     output_side_packet: "anchors"
     options {
@@ -89,7 +89,7 @@ TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
         fixed_anchor_size: true
       }
     }
-  )"));
+  )pb"));
 
   MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
 
@@ -106,7 +106,7 @@ TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
 }
 
 TEST(SsdAnchorCalculatorTest, MobileSSDConfig) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "SsdAnchorsCalculator"
     output_side_packet: "anchors"
     options {
@@ -132,7 +132,7 @@ TEST(SsdAnchorCalculatorTest, MobileSSDConfig) {
         reduce_boxes_in_lowest_layer: true
       }
     }
-  )"));
+  )pb"));
 
   MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const auto& anchors =

mediapipe/calculators/tflite/tflite_converter_calculator_test.cc

@@ -85,7 +85,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TfLiteConverterCalculator"
@@ -97,7 +97,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -146,7 +146,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
 
     // Run the calculator and verify that one output is generated.
     CalculatorGraphConfig graph_config =
-        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: "matrix"
           node {
             calculator: "TfLiteConverterCalculator"
@@ -158,7 +158,7 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
               }
             }
           }
-        )");
+        )pb");
     std::vector<Packet> output_packets;
     tool::AddVectorSink("tensor", &graph_config, &output_packets);
 
@@ -204,7 +204,7 @@ TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
   CalculatorGraph graph;
   // Run the calculator and verify that one output is generated.
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_image"
         node {
           calculator: "TfLiteConverterCalculator"
@@ -219,7 +219,7 @@ TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
             }
           }
         }
-      )");
+      )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor", &graph_config, &output_packets);
 

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -278,7 +278,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
 
   bool use_advanced_gpu_api_ = false;
   bool allow_precision_loss_ = false;
-  mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::API
+  mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api
       tflite_gpu_runner_api_;
 
   bool use_kernel_caching_ = false;
@@ -702,11 +702,16 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
 #if MEDIAPIPE_TFLITE_GL_INFERENCE
   ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
   const auto& model = *model_packet_.Get<TfLiteModelPtr>();
-  tflite::ops::builtin::BuiltinOpResolver op_resolver;
+
+  tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
+      default_op_resolver;
+  auto op_resolver_ptr =
+      static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
+          &default_op_resolver);
   if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
-    op_resolver = cc->InputSidePackets()
+    op_resolver_ptr = &(cc->InputSidePackets()
-                      .Tag("CUSTOM_OP_RESOLVER")
+                            .Tag("CUSTOM_OP_RESOLVER")
-                      .Get<tflite::ops::builtin::BuiltinOpResolver>();
+                            .Get<tflite::ops::builtin::BuiltinOpResolver>());
   }
 
   // Create runner
@@ -733,7 +738,7 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
     }
   }
   MP_RETURN_IF_ERROR(
-      tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
+      tflite_gpu_runner_->InitializeWithModel(model, *op_resolver_ptr));
 
   // Allocate interpreter memory for cpu output.
   if (!gpu_output_) {
@@ -786,18 +791,24 @@ absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) {
 
   ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc));
   const auto& model = *model_packet_.Get<TfLiteModelPtr>();
-  tflite::ops::builtin::BuiltinOpResolver op_resolver;
+
+  tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
+      default_op_resolver;
+  auto op_resolver_ptr =
+      static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
+          &default_op_resolver);
+
   if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
-    op_resolver = cc->InputSidePackets()
+    op_resolver_ptr = &(cc->InputSidePackets()
-                      .Tag("CUSTOM_OP_RESOLVER")
+                            .Tag("CUSTOM_OP_RESOLVER")
-                      .Get<tflite::ops::builtin::BuiltinOpResolver>();
+                            .Get<tflite::ops::builtin::BuiltinOpResolver>());
   }
 
 #if defined(MEDIAPIPE_EDGE_TPU)
   interpreter_ =
-      BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
+      BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get());
 #else
-  tflite::InterpreterBuilder(model, op_resolver)(&interpreter_);
+  tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
 #endif  // MEDIAPIPE_EDGE_TPU
 
   RET_CHECK(interpreter_);

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -51,12 +51,12 @@ message TfLiteInferenceCalculatorOptions {
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Choose any of available APIs to force running inference using it.
-      enum API {
+      enum Api {
         ANY = 0;
         OPENGL = 1;
         OPENCL = 2;
       }
-      optional API api = 4 [default = ANY];
+      optional Api api = 4 [default = ANY];
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Set to true to use 16-bit float precision. If max precision is needed,

mediapipe/calculators/tflite/tflite_model_calculator_test.cc

@@ -28,7 +28,7 @@ TEST(TfLiteModelCalculatorTest, SmokeTest) {
   // Prepare single calculator graph to and wait for packets.
   CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
       CalculatorGraphConfig>(
-      R"(
+      R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:model_path"
@@ -52,7 +52,7 @@ TEST(TfLiteModelCalculatorTest, SmokeTest) {
           input_side_packet: "MODEL_BLOB:model_blob"
           output_side_packet: "MODEL:model"
         }
-      )");
+      )pb");
   CalculatorGraph graph(graph_config);
   MP_ASSERT_OK(graph.StartRun({}));
   MP_ASSERT_OK(graph.WaitUntilIdle());

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc

@@ -73,14 +73,14 @@ class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TfLiteTensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
     options {
       [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {}
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -103,7 +103,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
 
 TEST_F(TfLiteTensorsToClassificationCalculatorTest,
        CorrectOutputWithLabelMapPath) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TfLiteTensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -112,7 +112,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
         label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt"
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -135,7 +135,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
 
 TEST_F(TfLiteTensorsToClassificationCalculatorTest,
        CorrectOutputWithLabelMinScoreThreshold) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TfLiteTensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -144,7 +144,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
         min_score_threshold: 0.6
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());
@@ -162,7 +162,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest,
 }
 
 TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
-  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
     calculator: "TfLiteTensorsToClassificationCalculator"
     input_stream: "TENSORS:tensors"
     output_stream: "CLASSIFICATIONS:classifications"
@@ -171,7 +171,7 @@ TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
         top_k: 2
       }
     }
-  )"));
+  )pb"));
 
   BuildGraph(&runner, {0, 0.5, 1});
   MP_ASSERT_OK(runner.Run());

mediapipe/calculators/util/BUILD

@@ -841,12 +841,39 @@ cc_library(
         "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util/filtering:one_euro_filter",
         "//mediapipe/util/filtering:relative_velocity_filter",
         "@com_google_absl//absl/algorithm:container",
     ],
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "visibility_smoothing_calculator_proto",
+    srcs = ["visibility_smoothing_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "visibility_smoothing_calculator",
+    srcs = ["visibility_smoothing_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":visibility_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util/filtering:low_pass_filter",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "landmarks_to_floats_calculator",
     srcs = ["landmarks_to_floats_calculator.cc"],
@@ -858,7 +885,7 @@ cc_library(
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -1194,3 +1221,34 @@ cc_library(
     }),
     alwayslink = 1,
 )
+
+cc_library(
+    name = "detection_classifications_merger_calculator",
+    srcs = ["detection_classifications_merger_calculator.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "detection_classifications_merger_calculator_test",
+    srcs = ["detection_classifications_merger_calculator_test.cc"],
+    deps = [
+        ":detection_classifications_merger_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:message_matchers",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+    ],
+)

mediapipe/calculators/util/association_calculator_test.cc

@@ -97,7 +97,7 @@ class AssociationDetectionCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationDetectionCalculator"
     input_stream: "input_vec_0"
     input_stream: "input_vec_1"
@@ -108,7 +108,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream 0: det_0, det_1, det_2.
   auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
@@ -160,7 +160,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
 }
 
 TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationDetectionCalculator"
     input_stream: "PREV:input_vec_0"
     input_stream: "input_vec_1"
@@ -170,7 +170,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream 0: det_3, det_4.
   auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
@@ -209,7 +209,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
 }
 
 TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationDetectionCalculator"
     input_stream: "input_vec_0"
     input_stream: "input_vec_1"
@@ -220,7 +220,7 @@ TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream 0: det_5.
   auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
@@ -332,7 +332,7 @@ class AssociationNormRectCalculatorTest : public ::testing::Test {
 };
 
 TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationNormRectCalculator"
     input_stream: "input_vec_0"
     input_stream: "input_vec_1"
@@ -343,7 +343,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream 0: nr_0, nr_1, nr_2.
   auto input_vec_0 =
@@ -386,7 +386,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
 }
 
 TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationNormRectCalculator"
     input_stream: "input_vec_0"
     input_stream: "input_vec_1"
@@ -397,7 +397,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream 0: nr_5.
   auto input_vec_0 =
@@ -441,7 +441,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
 }
 
 TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "AssociationNormRectCalculator"
     input_stream: "input_vec"
     output_stream: "output_vec"
@@ -450,7 +450,7 @@ TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
         min_similarity_threshold: 0.1
       }
     }
-  )"));
+  )pb"));
 
   // Input Stream : nr_3, nr_5.
   auto input_vec =

mediapipe/calculators/util/collection_has_min_size_calculator_test.cc

@@ -41,14 +41,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
 
 TEST(TestIntCollectionHasMinSizeCalculator, DoesHaveMinSize) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestIntCollectionHasMinSizeCalculator"
         input_stream: "ITERABLE:input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 2 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
 
@@ -70,12 +70,12 @@ TEST(TestIntCollectionHasMinSizeCalculator, DoesHaveMinSize) {
 TEST(TestIntCollectionHasMinSizeCalculator,
      DoesHaveMinSize_MinSizeAsSidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestIntCollectionHasMinSizeCalculator"
         input_stream: "ITERABLE:input_vector"
         input_side_packet: "min_size"
         output_stream: "output_vector"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
 
@@ -98,14 +98,14 @@ TEST(TestIntCollectionHasMinSizeCalculator,
 
 TEST(TestIntCollectionHasMinSizeCalculator, DoesNotHaveMinSize) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestIntCollectionHasMinSizeCalculator"
         input_stream: "ITERABLE:input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 3 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
 
@@ -127,12 +127,12 @@ TEST(TestIntCollectionHasMinSizeCalculator, DoesNotHaveMinSize) {
 TEST(TestIntCollectionHasMinSizeCalculator,
      DoesNotHaveMinSize_MinSizeAsSidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestIntCollectionHasMinSizeCalculator"
         input_stream: "ITERABLE:input_vector"
         input_side_packet: "min_size"
         output_stream: "output_vector"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
 

mediapipe/calculators/util/detection_classifications_merger_calculator.cc

@@ -0,0 +1,149 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/statusor.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {}  // namespace
+
+// Replaces the classification labels and scores from the input `Detection` with
+// the ones provided into the input `ClassificationList`. Namely:
+// * `label_id[i]` becomes `classification[i].index`
+// * `score[i]` becomes `classification[i].score`
+// * `label[i]` becomes `classification[i].label` (if present)
+//
+// In case the input `ClassificationList` contains no results (i.e.
+// `classification` is empty, which may happen if the classifier uses a score
+// threshold and no confident enough result were returned), the input
+// `Detection` is returned unchanged.
+//
+// This is specifically designed for two-stage detection cascades where the
+// detections returned by a standalone detector (typically a class-agnostic
+// localizer) are fed e.g. into a `TfLiteTaskImageClassifierCalculator` through
+// the optional "RECT" or "NORM_RECT" input, e.g:
+//
+// node {
+//   calculator: "DetectionsToRectsCalculator"
+//   # Output of an upstream object detector.
+//   input_stream: "DETECTION:detection"
+//   output_stream: "NORM_RECT:norm_rect"
+// }
+// node {
+//   calculator: "TfLiteTaskImageClassifierCalculator"
+//   input_stream: "IMAGE:image"
+//   input_stream: "NORM_RECT:norm_rect"
+//   output_stream: "CLASSIFICATION_RESULT:classification_result"
+// }
+// node {
+//   calculator: "TfLiteTaskClassificationResultToClassificationsCalculator"
+//   input_stream: "CLASSIFICATION_RESULT:classification_result"
+//   output_stream: "CLASSIFICATION_LIST:classification_list"
+// }
+// node {
+//   calculator: "DetectionClassificationsMergerCalculator"
+//   input_stream: "INPUT_DETECTION:detection"
+//   input_stream: "CLASSIFICATION_LIST:classification_list"
+//   # Final output.
+//   output_stream: "OUTPUT_DETECTION:classified_detection"
+// }
+//
+// Inputs:
+// INPUT_DETECTION: `Detection` proto.
+// CLASSIFICATION_LIST: `ClassificationList` proto.
+//
+// Output:
+// OUTPUT_DETECTION: modified `Detection` proto.
+class DetectionClassificationsMergerCalculator : public Node {
+ public:
+  static constexpr Input<Detection> kInputDetection{"INPUT_DETECTION"};
+  static constexpr Input<ClassificationList> kClassificationList{
+      "CLASSIFICATION_LIST"};
+  static constexpr Output<Detection> kOutputDetection{"OUTPUT_DETECTION"};
+
+  MEDIAPIPE_NODE_CONTRACT(kInputDetection, kClassificationList,
+                          kOutputDetection);
+
+  absl::Status Process(CalculatorContext* cc) override;
+};
+MEDIAPIPE_REGISTER_NODE(DetectionClassificationsMergerCalculator);
+
+absl::Status DetectionClassificationsMergerCalculator::Process(
+    CalculatorContext* cc) {
+  if (kInputDetection(cc).IsEmpty() && kClassificationList(cc).IsEmpty()) {
+    return absl::OkStatus();
+  }
+  RET_CHECK(!kInputDetection(cc).IsEmpty());
+  RET_CHECK(!kClassificationList(cc).IsEmpty());
+
+  Detection detection = *kInputDetection(cc);
+  const ClassificationList& classification_list = *kClassificationList(cc);
+
+  // Update input detection only if classification did return results.
+  if (classification_list.classification_size() != 0) {
+    detection.clear_label_id();
+    detection.clear_score();
+    detection.clear_label();
+    detection.clear_display_name();
+    for (const auto& classification : classification_list.classification()) {
+      if (!classification.has_index()) {
+        return absl::InvalidArgumentError(
+            "Missing required 'index' field in Classification proto.");
+      }
+      detection.add_label_id(classification.index());
+      if (!classification.has_score()) {
+        return absl::InvalidArgumentError(
+            "Missing required 'score' field in Classification proto.");
+      }
+      detection.add_score(classification.score());
+      if (classification.has_label()) {
+        detection.add_label(classification.label());
+      }
+      if (classification.has_display_name()) {
+        detection.add_display_name(classification.display_name());
+      }
+    }
+    // Post-conversion sanity checks.
+    if (detection.label_size() != 0 &&
+        detection.label_size() != detection.label_id_size()) {
+      return absl::InvalidArgumentError(absl::Substitute(
+          "Each input Classification is expected to either always or never "
+          "provide a 'label' field. Found $0 'label' fields for $1 "
+          "'Classification' objects.",
+          /*$0=*/detection.label_size(), /*$1=*/detection.label_id_size()));
+    }
+    if (detection.display_name_size() != 0 &&
+        detection.display_name_size() != detection.label_id_size()) {
+      return absl::InvalidArgumentError(absl::Substitute(
+          "Each input Classification is expected to either always or never "
+          "provide a 'display_name' field. Found $0 'display_name' fields for "
+          "$1 'Classification' objects.",
+          /*$0=*/detection.display_name_size(),
+          /*$1=*/detection.label_id_size()));
+    }
+  }
+  kOutputDetection(cc).Send(detection);
+  return absl::OkStatus();
+}
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc

@@ -0,0 +1,320 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/message_matchers.h"
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+constexpr char kGraphConfig[] = R"(
+        input_stream: "input_detection"
+        input_stream: "classification_list"
+        output_stream: "output_detection"
+        node {
+          calculator: "DetectionClassificationsMergerCalculator"
+          input_stream: "INPUT_DETECTION:input_detection"
+          input_stream: "CLASSIFICATION_LIST:classification_list"
+          output_stream: "OUTPUT_DETECTION:output_detection"
+        }
+      )";
+
+constexpr char kInputDetection[] = R"(
+        label: "entity"
+        label_id: 1
+        score: 0.9
+        location_data {
+          format: BOUNDING_BOX
+          bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
+        }
+        display_name: "Entity"
+     )";
+
+// Checks that the input Detection is returned unchanged if the input
+// ClassificationList does not contain any result.
+TEST(DetectionClassificationsMergerCalculator, SucceedsWithNoClassification) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>("");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Get and validate output.
+  EXPECT_THAT(output_packets, testing::SizeIs(1));
+  const Detection& output_detection = output_packets[0].Get<Detection>();
+  EXPECT_THAT(output_detection, mediapipe::EqualsProto(input_detection));
+}
+
+// Checks that merging succeeds when the input ClassificationList includes
+// labels and display names.
+TEST(DetectionClassificationsMergerCalculator,
+     SucceedsWithLabelsAndDisplayNames) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
+        classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Get and validate output.
+  EXPECT_THAT(output_packets, testing::SizeIs(1));
+  const Detection& output_detection = output_packets[0].Get<Detection>();
+  EXPECT_THAT(output_detection,
+              mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"pb(
+                label: "dog"
+                label: "fox"
+                label_id: 11
+                label_id: 12
+                score: 0.5
+                score: 0.4
+                location_data {
+                  format: BOUNDING_BOX
+                  bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
+                }
+                display_name: "Dog"
+                display_name: "Fox"
+              )pb")));
+}
+
+// Checks that merging succeeds when the input ClassificationList doesn't
+// include labels and display names.
+TEST(DetectionClassificationsMergerCalculator,
+     SucceedsWithoutLabelsAndDisplayNames) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { index: 11 score: 0.5 }
+        classification { index: 12 score: 0.4 }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Get and validate output.
+  EXPECT_THAT(output_packets, testing::SizeIs(1));
+  const Detection& output_detection = output_packets[0].Get<Detection>();
+  EXPECT_THAT(output_detection,
+              mediapipe::EqualsProto(ParseTextProtoOrDie<Detection>(R"pb(
+                label_id: 11
+                label_id: 12
+                score: 0.5
+                score: 0.4
+                location_data {
+                  format: BOUNDING_BOX
+                  bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 }
+                }
+              )pb")));
+}
+
+// Checks that merging fails if the input ClassificationList misses mandatory
+// "index" field.
+TEST(DetectionClassificationsMergerCalculator, FailsWithMissingIndex) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { score: 0.5 label: "dog" }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
+}
+
+// Checks that merging fails if the input ClassificationList misses mandatory
+// "score" field.
+TEST(DetectionClassificationsMergerCalculator, FailsWithMissingScore) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { index: 11 label: "dog" }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
+}
+
+// Checks that merging fails if the input ClassificationList has an
+// inconsistent number of labels.
+TEST(DetectionClassificationsMergerCalculator,
+     FailsWithInconsistentNumberOfLabels) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" }
+        classification { index: 12 score: 0.4 display_name: "Fox" }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
+}
+
+// Checks that merging fails if the input ClassificationList has an
+// inconsistent number of display names.
+TEST(DetectionClassificationsMergerCalculator,
+     FailsWithInconsistentNumberOfDisplayNames) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(kGraphConfig);
+
+  // Prepare input packets.
+  const Detection& input_detection =
+      ParseTextProtoOrDie<Detection>(kInputDetection);
+  Packet input_detection_packet =
+      MakePacket<Detection>(input_detection).At(Timestamp(0));
+  const ClassificationList& classification_list =
+      ParseTextProtoOrDie<ClassificationList>(R"pb(
+        classification { index: 11 score: 0.5 label: "dog" }
+        classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" }
+      )pb");
+  Packet classification_list_packet =
+      MakePacket<ClassificationList>(classification_list).At(Timestamp(0));
+
+  // Catch output.
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_detection", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(
+      graph.AddPacketToInputStream("input_detection", input_detection_packet));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list",
+                                            classification_list_packet));
+  ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/util/detection_letterbox_removal_calculator_test.cc

@@ -57,12 +57,12 @@ Detection CreateDetection(const std::vector<std::string>& labels,
 }
 
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionLetterboxRemovalCalculator"
     input_stream: "DETECTIONS:detections"
     input_stream: "LETTERBOX_PADDING:letterbox_padding"
     output_stream: "DETECTIONS:adjusted_detections"
-  )");
+  )pb");
 }
 
 TEST(DetectionLetterboxRemovalCalculatorTest, PaddingLeftRight) {

mediapipe/calculators/util/detection_projection_calculator_test.cc

@@ -66,12 +66,12 @@ std::vector<Point2_f> GetPoints(const Detection& detection) {
 // Test helper function to run "DetectionProjectionCalculator".
 absl::StatusOr<Detection> RunProjectionCalculator(
     Detection detection, std::array<float, 16> project_mat) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionProjectionCalculator"
     input_stream: "DETECTIONS:detections"
     input_stream: "PROJECTION_MATRIX:matrix"
     output_stream: "DETECTIONS:projected_detections"
-  )"));
+  )pb"));
 
   runner.MutableInputs()
       ->Tag("DETECTIONS")

mediapipe/calculators/util/detections_to_rects_calculator_test.cc

@@ -84,11 +84,11 @@ Detection DetectionWithRelativeLocationData(double xmin, double ymin,
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionToRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     output_stream: "RECT:rect"
-  )"));
+  )pb"));
 
   auto detection = absl::make_unique<Detection>(
       DetectionWithLocationData(100, 200, 300, 400));
@@ -107,7 +107,7 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToRect) {
 
 absl::StatusOr<Rect> RunDetectionKeyPointsToRectCalculation(
     Detection detection, std::pair<int, int> image_size) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     input_stream: "IMAGE_SIZE:image_size"
@@ -117,7 +117,7 @@ absl::StatusOr<Rect> RunDetectionKeyPointsToRectCalculation(
         conversion_mode: USE_KEYPOINTS
       }
     }
-  )"));
+  )pb"));
 
   runner.MutableInputs()
       ->Tag("DETECTION")
@@ -160,11 +160,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToRect) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     output_stream: "NORM_RECT:rect"
-  )"));
+  )pb"));
 
   auto detection = absl::make_unique<Detection>(
       DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
@@ -183,7 +183,7 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRect) {
 
 absl::StatusOr<NormalizedRect> RunDetectionKeyPointsToNormRectCalculation(
     Detection detection) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     output_stream: "NORM_RECT:rect"
@@ -192,7 +192,7 @@ absl::StatusOr<NormalizedRect> RunDetectionKeyPointsToNormRectCalculation(
         conversion_mode: USE_KEYPOINTS
       }
     }
-  )"));
+  )pb"));
 
   runner.MutableInputs()
       ->Tag("DETECTION")
@@ -231,11 +231,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToNormalizedRect) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionsToRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "RECT:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithLocationData(100, 200, 300, 400));
@@ -254,11 +254,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToRect) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "NORM_RECT:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
@@ -277,11 +277,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRect) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionsToRects) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "RECTS:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithLocationData(100, 200, 300, 400));
@@ -302,11 +302,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToRects) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRects) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "NORM_RECTS:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
@@ -328,11 +328,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRects) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionToRects) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     output_stream: "RECTS:rect"
-  )"));
+  )pb"));
 
   auto detection = absl::make_unique<Detection>(
       DetectionWithLocationData(100, 200, 300, 400));
@@ -351,11 +351,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToRects) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRects) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTION:detection"
     output_stream: "NORM_RECTS:rect"
-  )"));
+  )pb"));
 
   auto detection = absl::make_unique<Detection>(
       DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
@@ -375,11 +375,11 @@ TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRects) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, WrongInputToRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "RECT:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4));
@@ -395,11 +395,11 @@ TEST(DetectionsToRectsCalculatorTest, WrongInputToRect) {
 }
 
 TEST(DetectionsToRectsCalculatorTest, WrongInputToNormalizedRect) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRectsCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "NORM_RECT:rect"
-  )"));
+  )pb"));
 
   auto detections(absl::make_unique<std::vector<Detection>>());
   detections->push_back(DetectionWithLocationData(100, 200, 300, 400));

mediapipe/calculators/util/detections_to_render_data_calculator_test.cc

@@ -85,11 +85,11 @@ Detection CreateDetection(const std::vector<std::string>& labels,
 }
 
 TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionList) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRenderDataCalculator"
     input_stream: "DETECTION_LIST:detection_list"
     output_stream: "RENDER_DATA:render_data"
-  )"));
+  )pb"));
 
   LocationData location_data = CreateLocationData(100, 200, 300, 400);
   auto detections(absl::make_unique<DetectionList>());
@@ -119,11 +119,11 @@ TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionList) {
 }
 
 TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionVector) {
-  CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRenderDataCalculator"
     input_stream: "DETECTIONS:detections"
     output_stream: "RENDER_DATA:render_data"
-  )")};
+  )pb")};
 
   LocationData location_data = CreateLocationData(100, 200, 300, 400);
   auto detections(absl::make_unique<std::vector<Detection>>());
@@ -153,12 +153,12 @@ TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionVector) {
 }
 
 TEST(DetectionsToRenderDataCalculatorTest, BothDetecctionListAndVector) {
-  CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "DetectionsToRenderDataCalculator"
     input_stream: "DETECTION_LIST:detection_list"
     input_stream: "DETECTIONS:detections"
     output_stream: "RENDER_DATA:render_data"
-  )")};
+  )pb")};
 
   LocationData location_data1 = CreateLocationData(100, 200, 300, 400);
   auto detection_list(absl::make_unique<DetectionList>());
@@ -194,17 +194,18 @@ TEST(DetectionsToRenderDataCalculatorTest, BothDetecctionListAndVector) {
 
 TEST(DetectionsToRenderDataCalculatorTest, ProduceEmptyPacket) {
   // Check when produce_empty_packet is false.
-  CalculatorRunner runner1{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner1{
-    calculator: "DetectionsToRenderDataCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "DETECTION_LIST:detection_list"
+        calculator: "DetectionsToRenderDataCalculator"
-    input_stream: "DETECTIONS:detections"
+        input_stream: "DETECTION_LIST:detection_list"
-    output_stream: "RENDER_DATA:render_data"
+        input_stream: "DETECTIONS:detections"
-    options {
+        output_stream: "RENDER_DATA:render_data"
-      [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
+        options {
-        produce_empty_packet: false
+          [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
-      }
+            produce_empty_packet: false
-    }
+          }
-  )")};
+        }
+      )pb")};
 
   auto detection_list1(absl::make_unique<DetectionList>());
   runner1.MutableInputs()
@@ -224,17 +225,18 @@ TEST(DetectionsToRenderDataCalculatorTest, ProduceEmptyPacket) {
   ASSERT_EQ(0, exact1.size());
 
   // Check when produce_empty_packet is true.
-  CalculatorRunner runner2{ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner2{
-    calculator: "DetectionsToRenderDataCalculator"
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
-    input_stream: "DETECTION_LIST:detection_list"
+        calculator: "DetectionsToRenderDataCalculator"
-    input_stream: "DETECTIONS:detections"
+        input_stream: "DETECTION_LIST:detection_list"
-    output_stream: "RENDER_DATA:render_data"
+        input_stream: "DETECTIONS:detections"
-    options {
+        output_stream: "RENDER_DATA:render_data"
-      [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
+        options {
-        produce_empty_packet: true
+          [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
-      }
+            produce_empty_packet: true
-    }
+          }
-  )")};
+        }
+      )pb")};
 
   auto detection_list2(absl::make_unique<DetectionList>());
   runner2.MutableInputs()

mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc

@@ -32,12 +32,12 @@ NormalizedLandmark CreateLandmark(float x, float y) {
 }
 
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "LandmarkLetterboxRemovalCalculator"
     input_stream: "LANDMARKS:landmarks"
     input_stream: "LETTERBOX_PADDING:letterbox_padding"
     output_stream: "LANDMARKS:adjusted_landmarks"
-  )");
+  )pb");
 }
 
 TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) {

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -12,12 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <memory>
+
 #include "absl/algorithm/container.h"
 #include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
 #include "mediapipe/util/filtering/relative_velocity_filter.h"
 
 namespace mediapipe {
@@ -25,19 +28,54 @@ namespace mediapipe {
 namespace {
 
 constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
+constexpr char kLandmarksTag[] = "LANDMARKS";
 constexpr char kImageSizeTag[] = "IMAGE_SIZE";
 constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
+constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
 
+using mediapipe::OneEuroFilter;
 using mediapipe::RelativeVelocityFilter;
 
+void NormalizedLandmarksToLandmarks(
+    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+    const int image_height, LandmarkList* landmarks) {
+  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
+    const auto& norm_landmark = norm_landmarks.landmark(i);
+
+    auto* landmark = landmarks->add_landmark();
+    landmark->set_x(norm_landmark.x() * image_width);
+    landmark->set_y(norm_landmark.y() * image_height);
+    // Scale Z the same way as X (using image width).
+    landmark->set_z(norm_landmark.z() * image_width);
+    landmark->set_visibility(norm_landmark.visibility());
+    landmark->set_presence(norm_landmark.presence());
+  }
+}
+
+void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
+                                    const int image_width,
+                                    const int image_height,
+                                    NormalizedLandmarkList* norm_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+
+    auto* norm_landmark = norm_landmarks->add_landmark();
+    norm_landmark->set_x(landmark.x() / image_width);
+    norm_landmark->set_y(landmark.y() / image_height);
+    // Scale Z the same way as X (using image width).
+    norm_landmark->set_z(landmark.z() / image_width);
+    norm_landmark->set_visibility(landmark.visibility());
+    norm_landmark->set_presence(landmark.presence());
+  }
+}
+
 // Estimate object scale to use its inverse value as velocity scale for
 // RelativeVelocityFilter. If value will be too small (less than
 // `options_.min_allowed_object_scale`) smoothing will be disabled and
 // landmarks will be returned as is.
 // Object scale is calculated as average between bounding box width and height
 // with sides parallel to axis.
-float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
+float GetObjectScale(const LandmarkList& landmarks) {
-                     int image_height) {
   const auto& lm_minmax_x = absl::c_minmax_element(
       landmarks.landmark(),
       [](const auto& a, const auto& b) { return a.x() < b.x(); });
@@ -50,8 +88,8 @@ float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
   const float y_min = lm_minmax_y.first->y();
   const float y_max = lm_minmax_y.second->y();
 
-  const float object_width = (x_max - x_min) * image_width;
+  const float object_width = x_max - x_min;
-  const float object_height = (y_max - y_min) * image_height;
+  const float object_height = y_max - y_min;
 
   return (object_width + object_height) / 2.0f;
 }
@@ -63,19 +101,17 @@ class LandmarksFilter {
 
   virtual absl::Status Reset() { return absl::OkStatus(); }
 
-  virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+  virtual absl::Status Apply(const LandmarkList& in_landmarks,
-                             const std::pair<int, int>& image_size,
                              const absl::Duration& timestamp,
-                             NormalizedLandmarkList* out_landmarks) = 0;
+                             LandmarkList* out_landmarks) = 0;
 };
 
 // Returns landmarks as is without smoothing.
 class NoFilter : public LandmarksFilter {
  public:
-  absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const std::pair<int, int>& image_size,
                      const absl::Duration& timestamp,
-                     NormalizedLandmarkList* out_landmarks) override {
+                     LandmarkList* out_landmarks) override {
     *out_landmarks = in_landmarks;
     return absl::OkStatus();
   }
@@ -85,10 +121,11 @@ class NoFilter : public LandmarksFilter {
 class VelocityFilter : public LandmarksFilter {
  public:
   VelocityFilter(int window_size, float velocity_scale,
-                 float min_allowed_object_scale)
+                 float min_allowed_object_scale, bool disable_value_scaling)
       : window_size_(window_size),
         velocity_scale_(velocity_scale),
-        min_allowed_object_scale_(min_allowed_object_scale) {}
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
 
   absl::Status Reset() override {
     x_filters_.clear();
@@ -97,45 +134,37 @@ class VelocityFilter : public LandmarksFilter {
     return absl::OkStatus();
   }
 
-  absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const std::pair<int, int>& image_size,
                      const absl::Duration& timestamp,
-                     NormalizedLandmarkList* out_landmarks) override {
+                     LandmarkList* out_landmarks) override {
-    // Get image size.
-    int image_width;
-    int image_height;
-    std::tie(image_width, image_height) = image_size;
-
     // Get value scale as inverse value of the object scale.
     // If value is too small smoothing will be disabled and landmarks will be
     // returned as is.
-    const float object_scale =
+    float value_scale = 1.0f;
-        GetObjectScale(in_landmarks, image_width, image_height);
+    if (!disable_value_scaling_) {
-    if (object_scale < min_allowed_object_scale_) {
+      const float object_scale = GetObjectScale(in_landmarks);
-      *out_landmarks = in_landmarks;
+      if (object_scale < min_allowed_object_scale_) {
-      return absl::OkStatus();
+        *out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
     }
-    const float value_scale = 1.0f / object_scale;
 
     // Initialize filters once.
     MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
 
     // Filter landmarks. Every axis of every landmark is filtered separately.
     for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const NormalizedLandmark& in_landmark = in_landmarks.landmark(i);
+      const auto& in_landmark = in_landmarks.landmark(i);
 
-      NormalizedLandmark* out_landmark = out_landmarks->add_landmark();
+      auto* out_landmark = out_landmarks->add_landmark();
       *out_landmark = in_landmark;
-      out_landmark->set_x(x_filters_[i].Apply(timestamp, value_scale,
+      out_landmark->set_x(
-                                              in_landmark.x() * image_width) /
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-                          image_width);
+      out_landmark->set_y(
-      out_landmark->set_y(y_filters_[i].Apply(timestamp, value_scale,
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-                                              in_landmark.y() * image_height) /
+      out_landmark->set_z(
-                          image_height);
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-      // Scale Z the save was as X (using image width).
-      out_landmark->set_z(z_filters_[i].Apply(timestamp, value_scale,
-                                              in_landmark.z() * image_width) /
-                          image_width);
     }
 
     return absl::OkStatus();
@@ -165,12 +194,83 @@ class VelocityFilter : public LandmarksFilter {
   int window_size_;
   float velocity_scale_;
   float min_allowed_object_scale_;
+  bool disable_value_scaling_;
 
   std::vector<RelativeVelocityFilter> x_filters_;
   std::vector<RelativeVelocityFilter> y_filters_;
   std::vector<RelativeVelocityFilter> z_filters_;
 };
 
+// Please check OneEuroFilter documentation for details.
+class OneEuroFilterImpl : public LandmarksFilter {
+ public:
+  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
+                    double derivate_cutoff)
+      : frequency_(frequency),
+        min_cutoff_(min_cutoff),
+        beta_(beta),
+        derivate_cutoff_(derivate_cutoff) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     LandmarkList* out_landmarks) override {
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks->add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(x_filters_[i].Apply(timestamp, in_landmark.x()));
+      out_landmark->set_y(y_filters_[i].Apply(timestamp, in_landmark.y()));
+      out_landmark->set_z(z_filters_[i].Apply(timestamp, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    for (int i = 0; i < n_landmarks; ++i) {
+      x_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      y_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      z_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+    }
+
+    return absl::OkStatus();
+  }
+
+  double frequency_;
+  double min_cutoff_;
+  double beta_;
+  double derivate_cutoff_;
+
+  std::vector<OneEuroFilter> x_filters_;
+  std::vector<OneEuroFilter> y_filters_;
+  std::vector<OneEuroFilter> z_filters_;
+};
+
 }  // namespace
 
 // A calculator to smooth landmarks over time.
@@ -207,16 +307,21 @@ class LandmarksSmoothingCalculator : public CalculatorBase {
   absl::Status Process(CalculatorContext* cc) override;
 
  private:
-  LandmarksFilter* landmarks_filter_;
+  std::unique_ptr<LandmarksFilter> landmarks_filter_;
 };
 REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
 
 absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
-  cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
+  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-  cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
+    cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
-  cc->Outputs()
+    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
-      .Tag(kNormalizedFilteredLandmarksTag)
+    cc->Outputs()
-      .Set<NormalizedLandmarkList>();
+        .Tag(kNormalizedFilteredLandmarksTag)
+        .Set<NormalizedLandmarkList>();
+  } else {
+    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
+    cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
+  }
 
   return absl::OkStatus();
 }
@@ -227,12 +332,19 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
   // Pick landmarks filter.
   const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
   if (options.has_no_filter()) {
-    landmarks_filter_ = new NoFilter();
+    landmarks_filter_ = absl::make_unique<NoFilter>();
   } else if (options.has_velocity_filter()) {
-    landmarks_filter_ = new VelocityFilter(
+    landmarks_filter_ = absl::make_unique<VelocityFilter>(
         options.velocity_filter().window_size(),
         options.velocity_filter().velocity_scale(),
-        options.velocity_filter().min_allowed_object_scale());
+        options.velocity_filter().min_allowed_object_scale(),
+        options.velocity_filter().disable_value_scaling());
+  } else if (options.has_one_euro_filter()) {
+    landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
+        options.one_euro_filter().frequency(),
+        options.one_euro_filter().min_cutoff(),
+        options.one_euro_filter().beta(),
+        options.one_euro_filter().derivate_cutoff());
   } else {
     RET_CHECK_FAIL()
         << "Landmarks filter is either not specified or not supported";
@@ -244,25 +356,53 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
 absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
   // Check that landmarks are not empty and reset the filter if so.
   // Don't emit an empty packet for this timestamp.
-  if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
+  if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
+       cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
+      (cc->Inputs().HasTag(kLandmarksTag) &&
+       cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
     MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
     return absl::OkStatus();
   }
 
-  const auto& in_landmarks =
-      cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
-  const auto& image_size =
-      cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
   const auto& timestamp =
       absl::Microseconds(cc->InputTimestamp().Microseconds());
 
-  auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
+  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-  MP_RETURN_IF_ERROR(landmarks_filter_->Apply(in_landmarks, image_size,
+    const auto& in_norm_landmarks =
-                                              timestamp, out_landmarks.get()));
+        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
 
-  cc->Outputs()
+    int image_width;
-      .Tag(kNormalizedFilteredLandmarksTag)
+    int image_height;
-      .Add(out_landmarks.release(), cc->InputTimestamp());
+    std::tie(image_width, image_height) =
+        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
+
+    auto in_landmarks = absl::make_unique<LandmarkList>();
+    NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
+                                   in_landmarks.get());
+
+    auto out_landmarks = absl::make_unique<LandmarkList>();
+    MP_RETURN_IF_ERROR(landmarks_filter_->Apply(*in_landmarks, timestamp,
+                                                out_landmarks.get()));
+
+    auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
+    LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
+                                   out_norm_landmarks.get());
+
+    cc->Outputs()
+        .Tag(kNormalizedFilteredLandmarksTag)
+        .Add(out_norm_landmarks.release(), cc->InputTimestamp());
+  } else {
+    const auto& in_landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
+
+    auto out_landmarks = absl::make_unique<LandmarkList>();
+    MP_RETURN_IF_ERROR(
+        landmarks_filter_->Apply(in_landmarks, timestamp, out_landmarks.get()));
+
+    cc->Outputs()
+        .Tag(kFilteredLandmarksTag)
+        .Add(out_landmarks.release(), cc->InputTimestamp());
+  }
 
   return absl::OkStatus();
 }

mediapipe/calculators/util/landmarks_smoothing_calculator.proto

@@ -39,10 +39,40 @@ message LandmarksSmoothingCalculatorOptions {
     // If calculated object scale is less than given value smoothing will be
     // disabled and landmarks will be returned as is.
     optional float min_allowed_object_scale = 3 [default = 1e-6];
+
+    // Disable value scaling based on object size and use `1.0` instead.
+    // Value scale is calculated as inverse value of object size. Object size is
+    // calculated as maximum side of rectangular bounding box of the object in
+    // XY plane.
+    optional bool disable_value_scaling = 4 [default = false];
+  }
+
+  // For the details of the filter implementation and the procedure of its
+  // configuration please check http://cristal.univ-lille.fr/~casiez/1euro/
+  message OneEuroFilter {
+    // Frequency of incomming frames defined in seconds. Used only if can't be
+    // calculated from provided events (e.g. on the very first frame).
+    optional float frequency = 1 [default = 0.033];
+
+    // Minimum cutoff frequency. Start by tuning this parameter while keeping
+    // `beta = 0` to reduce jittering to the desired level. 1Hz (the default
+    // value) is a good starting point.
+    optional float min_cutoff = 2 [default = 1.0];
+
+    // Cutoff slope. After `min_cutoff` is configured, start increasing `beta`
+    // value to reduce the lag introduced by the `min_cutoff`. Find the desired
+    // balance between jittering and lag.
+    optional float beta = 3 [default = 0.0];
+
+    // Cutoff frequency for derivate. It is set to 1Hz in the original
+    // algorithm, but can be tuned to further smooth the speed (i.e. derivate)
+    // on the object.
+    optional float derivate_cutoff = 4 [default = 1.0];
   }
 
   oneof filter_options {
     NoFilter no_filter = 1;
     VelocityFilter velocity_filter = 2;
+    OneEuroFilter one_euro_filter = 3;
   }
 }

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -34,6 +34,34 @@ constexpr char kRenderScaleTag[] = "RENDER_SCALE";
 constexpr char kRenderDataTag[] = "RENDER_DATA";
 constexpr char kLandmarkLabel[] = "KEYPOINT";
 
+inline Color DefaultMinDepthLineColor() {
+  Color color;
+  color.set_r(0);
+  color.set_g(0);
+  color.set_b(0);
+  return color;
+}
+
+inline Color DefaultMaxDepthLineColor() {
+  Color color;
+  color.set_r(255);
+  color.set_g(255);
+  color.set_b(255);
+  return color;
+}
+
+inline Color MixColors(const Color& color1, const Color& color2,
+                       float color1_weight) {
+  Color color;
+  color.set_r(static_cast<int>(color1.r() * color1_weight +
+                               color2.r() * (1.f - color1_weight)));
+  color.set_g(static_cast<int>(color1.g() * color1_weight +
+                               color2.g() * (1.f - color1_weight)));
+  color.set_b(static_cast<int>(color1.b() * color1_weight +
+                               color2.b() * (1.f - color1_weight)));
+  return color;
+}
+
 inline void SetColor(RenderAnnotation* annotation, const Color& color) {
   annotation->mutable_color()->set_r(color.r());
   annotation->mutable_color()->set_g(color.g());
@@ -57,6 +85,23 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
   }
 }
 
+template <class LandmarkType>
+bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark,
+                                  bool utilize_visibility,
+                                  float visibility_threshold,
+                                  bool utilize_presence,
+                                  float presence_threshold) {
+  if (utilize_visibility && landmark.has_visibility() &&
+      landmark.visibility() < visibility_threshold) {
+    return false;
+  }
+  if (utilize_presence && landmark.has_presence() &&
+      landmark.presence() < presence_threshold) {
+    return false;
+  }
+  return true;
+}
+
 void SetColorSizeValueFromZ(float z, float z_min, float z_max,
                             RenderAnnotation* render_annotation,
                             float min_depth_circle_thickness,
@@ -75,8 +120,9 @@ void SetColorSizeValueFromZ(float z, float z_min, float z_max,
 
 template <class LandmarkType>
 void AddConnectionToRenderData(const LandmarkType& start,
-                               const LandmarkType& end, int gray_val1,
+                               const LandmarkType& end,
-                               int gray_val2, float thickness, bool normalized,
+                               const Color& color_start, const Color& color_end,
+                               float thickness, bool normalized,
                                RenderData* render_data) {
   auto* connection_annotation = render_data->add_render_annotations();
   RenderAnnotation::GradientLine* line =
@@ -86,12 +132,13 @@ void AddConnectionToRenderData(const LandmarkType& start,
   line->set_x_end(end.x());
   line->set_y_end(end.y());
   line->set_normalized(normalized);
-  line->mutable_color1()->set_r(gray_val1);
+  line->mutable_color1()->set_r(color_start.r());
-  line->mutable_color1()->set_g(gray_val1);
+  line->mutable_color1()->set_g(color_start.g());
-  line->mutable_color1()->set_b(gray_val1);
+  line->mutable_color1()->set_b(color_start.b());
-  line->mutable_color2()->set_r(gray_val2);
+  line->mutable_color2()->set_r(color_end.r());
-  line->mutable_color2()->set_g(gray_val2);
+  line->mutable_color2()->set_g(color_end.g());
-  line->mutable_color2()->set_b(gray_val2);
+  line->mutable_color2()->set_b(color_end.b());
+
   connection_annotation->set_thickness(thickness);
 }
 
@@ -102,26 +149,26 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
                              float visibility_threshold, bool utilize_presence,
                              float presence_threshold, float thickness,
                              bool normalized, float min_z, float max_z,
+                             const Color& min_depth_line_color,
+                             const Color& max_depth_line_color,
                              RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (utilize_visibility &&
+    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
-        ((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
+            ld0, utilize_visibility, visibility_threshold, utilize_presence,
-         (ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
+            presence_threshold) ||
+        !IsLandmarkVisibileAndPresent<LandmarkType>(
+            ld1, utilize_visibility, visibility_threshold, utilize_presence,
+            presence_threshold)) {
       continue;
     }
-    if (utilize_presence &&
+    const Color color0 = MixColors(min_depth_line_color, max_depth_line_color,
-        ((ld0.has_presence() && ld0.presence() < presence_threshold) ||
+                                   Remap(ld0.z(), min_z, max_z, 1.f));
-         (ld1.has_presence() && ld1.presence() < presence_threshold))) {
+    const Color color1 = MixColors(min_depth_line_color, max_depth_line_color,
-      continue;
+                                   Remap(ld1.z(), min_z, max_z, 1.f));
-    }
+    AddConnectionToRenderData<LandmarkType>(ld0, ld1, color0, color1, thickness,
-    const int gray_val1 =
+                                            normalized, render_data);
-        255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
-    const int gray_val2 =
-        255 - static_cast<int>(Remap(ld1.z(), min_z, max_z, 255));
-    AddConnectionToRenderData<LandmarkType>(ld0, ld1, gray_val1, gray_val2,
-                                            thickness, normalized, render_data);
   }
 }
 
@@ -151,14 +198,12 @@ void AddConnections(const LandmarkListType& landmarks,
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (utilize_visibility &&
+    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
-        ((ld0.has_visibility() && ld0.visibility() < visibility_threshold) ||
+            ld0, utilize_visibility, visibility_threshold, utilize_presence,
-         (ld1.has_visibility() && ld1.visibility() < visibility_threshold))) {
+            presence_threshold) ||
-      continue;
+        !IsLandmarkVisibileAndPresent<LandmarkType>(
-    }
+            ld1, utilize_visibility, visibility_threshold, utilize_presence,
-    if (utilize_presence &&
+            presence_threshold)) {
-        ((ld0.has_presence() && ld0.presence() < presence_threshold) ||
-         (ld1.has_presence() && ld1.presence() < presence_threshold))) {
       continue;
     }
     AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
@@ -232,6 +277,13 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
   float z_min = 0.f;
   float z_max = 0.f;
 
+  const Color min_depth_line_color = options_.has_min_depth_line_color()
+                                         ? options_.min_depth_line_color()
+                                         : DefaultMinDepthLineColor();
+  const Color max_depth_line_color = options_.has_max_depth_line_color()
+                                         ? options_.max_depth_line_color()
+                                         : DefaultMaxDepthLineColor();
+
   // Apply scale to `thickness` of rendered landmarks and connections to make
   // them bigger when object (e.g. pose, hand or face) is closer/bigger and
   // snaller when object is further/smaller.
@@ -254,7 +306,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
           landmarks, landmark_connections_, options_.utilize_visibility(),
           options_.visibility_threshold(), options_.utilize_presence(),
           options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
-          z_max, render_data.get());
+          z_max, min_depth_line_color, max_depth_line_color, render_data.get());
     } else {
       AddConnections<LandmarkList, Landmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
@@ -265,13 +317,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
 
-      if (options_.utilize_visibility() && landmark.has_visibility() &&
+      if (!IsLandmarkVisibileAndPresent<Landmark>(
-          landmark.visibility() < options_.visibility_threshold()) {
+              landmark, options_.utilize_visibility(),
-        continue;
+              options_.visibility_threshold(), options_.utilize_presence(),
-      }
+              options_.presence_threshold())) {
-
-      if (options_.utilize_presence() && landmark.has_presence() &&
-          landmark.presence() < options_.presence_threshold()) {
         continue;
       }
 
@@ -303,7 +352,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
           landmarks, landmark_connections_, options_.utilize_visibility(),
           options_.visibility_threshold(), options_.utilize_presence(),
           options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
-          z_max, render_data.get());
+          z_max, min_depth_line_color, max_depth_line_color, render_data.get());
     } else {
       AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
@@ -314,12 +363,10 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
 
-      if (options_.utilize_visibility() && landmark.has_visibility() &&
+      if (!IsLandmarkVisibileAndPresent<NormalizedLandmark>(
-          landmark.visibility() < options_.visibility_threshold()) {
+              landmark, options_.utilize_visibility(),
-        continue;
+              options_.visibility_threshold(), options_.utilize_presence(),
-      }
+              options_.presence_threshold())) {
-      if (options_.utilize_presence() && landmark.has_presence() &&
-          landmark.presence() < options_.presence_threshold()) {
         continue;
       }
 

mediapipe/calculators/util/landmarks_to_render_data_calculator.proto

@@ -64,4 +64,10 @@ message LandmarksToRenderDataCalculatorOptions {
 
   // Max thickness of the drawing for landmark circle.
   optional double max_depth_circle_thickness = 11 [default = 18.0];
+
+  // Gradient color for the lines connecting landmarks at the minimum depth.
+  optional Color min_depth_line_color = 12;
+
+  // Gradient color for the lines connecting landmarks at the maximum depth.
+  optional Color max_depth_line_color = 13;
 }

mediapipe/calculators/util/packet_frequency_calculator_test.cc

@@ -24,7 +24,7 @@ namespace mediapipe {
 namespace {
 
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "PacketFrequencyCalculator"
     input_stream: "packet_stream"
     output_stream: "packet_frequency"
@@ -34,11 +34,11 @@ CalculatorGraphConfig::Node GetDefaultNode() {
         label: "stream_description"
       }
     }
-  )");
+  )pb");
 }
 
 CalculatorGraphConfig::Node GetNodeWithMultipleStreams() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "PacketFrequencyCalculator"
     input_stream: "packet_stream_0"
     input_stream: "packet_stream_1"
@@ -55,7 +55,7 @@ CalculatorGraphConfig::Node GetNodeWithMultipleStreams() {
         label: "stream_description_2"
       }
     }
-  )");
+  )pb");
 }
 
 // Tests packet frequency.

mediapipe/calculators/util/packet_latency_calculator_test.cc

@@ -38,7 +38,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
   }
 
   void InitializeSingleStreamGraph() {
-    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: "delayed_packet_0"
       input_stream: "camera_frames"
       node {
@@ -59,7 +59,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
           input_stream_handler: "ImmediateInputStreamHandler"
         }
       }
-    )");
+    )pb");
 
     mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
                                    &out_0_packets_);
@@ -79,7 +79,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
   }
 
   void InitializeMultipleStreamGraph() {
-    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: "delayed_packet_0"
       input_stream: "delayed_packet_1"
       input_stream: "delayed_packet_2"
@@ -107,7 +107,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
           input_stream_handler: "ImmediateInputStreamHandler"
         }
       }
-    )");
+    )pb");
 
     mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
                                    &out_0_packets_);
@@ -131,7 +131,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
   }
 
   void InitializeSingleStreamGraphWithoutClock() {
-    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: "delayed_packet_0"
       input_stream: "camera_frames"
       node {
@@ -150,7 +150,7 @@ class PacketLatencyCalculatorTest : public ::testing::Test {
           input_stream_handler: "ImmediateInputStreamHandler"
         }
       }
-    )");
+    )pb");
 
     mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_,
                                    &out_0_packets_);

mediapipe/calculators/util/top_k_scores_calculator_test.cc

@@ -24,7 +24,7 @@
 namespace mediapipe {
 
 TEST(TopKScoresCalculatorTest, TestNodeConfig) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "TopKScoresCalculator"
     input_stream: "SCORES:score_vector"
     output_stream: "TOP_K_INDEXES:top_k_indexes"
@@ -32,7 +32,7 @@ TEST(TopKScoresCalculatorTest, TestNodeConfig) {
     options: {
       [mediapipe.TopKScoresCalculatorOptions.ext] {}
     }
-  )"));
+  )pb"));
 
   auto status = runner.Run();
   ASSERT_TRUE(!status.ok());
@@ -43,7 +43,7 @@ TEST(TopKScoresCalculatorTest, TestNodeConfig) {
 }
 
 TEST(TopKScoresCalculatorTest, TestTopKOnly) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "TopKScoresCalculator"
     input_stream: "SCORES:score_vector"
     output_stream: "TOP_K_INDEXES:top_k_indexes"
@@ -51,7 +51,7 @@ TEST(TopKScoresCalculatorTest, TestTopKOnly) {
     options: {
       [mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 2 }
     }
-  )"));
+  )pb"));
 
   std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
 
@@ -76,7 +76,7 @@ TEST(TopKScoresCalculatorTest, TestTopKOnly) {
 }
 
 TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "TopKScoresCalculator"
     input_stream: "SCORES:score_vector"
     output_stream: "TOP_K_INDEXES:top_k_indexes"
@@ -84,7 +84,7 @@ TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
     options: {
       [mediapipe.TopKScoresCalculatorOptions.ext] { threshold: 0.2 }
     }
-  )"));
+  )pb"));
 
   std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
 
@@ -113,7 +113,7 @@ TEST(TopKScoresCalculatorTest, TestThresholdOnly) {
 }
 
 TEST(TopKScoresCalculatorTest, TestBothTopKAndThreshold) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "TopKScoresCalculator"
     input_stream: "SCORES:score_vector"
     output_stream: "TOP_K_INDEXES:top_k_indexes"
@@ -121,7 +121,7 @@ TEST(TopKScoresCalculatorTest, TestBothTopKAndThreshold) {
     options: {
       [mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 4 threshold: 0.3 }
     }
-  )"));
+  )pb"));
 
   std::vector<float> score_vector{0.9, 0.2, 0.3, 1.0, 0.1};
 

mediapipe/calculators/util/visibility_copy_calculator.cc

@@ -0,0 +1,194 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+
+#include "absl/algorithm/container.h"
+#include "mediapipe/calculators/util/visibility_copy_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kLandmarksFromTag[] = "LANDMARKS_FROM";
+constexpr char kNormalizedLandmarksFromTag[] = "NORM_LANDMARKS_FROM";
+constexpr char kLandmarksToTag[] = "LANDMARKS_TO";
+constexpr char kNormalizedLandmarksToTag[] = "NORM_LANDMARKS_TO";
+
+}  // namespace
+
+// A calculator to copy visibility and presence between landmarks.
+//
+// Landmarks to copy from and to copy to can be of different type (normalized or
+// non-normalized), but ladnmarks to copy to and output landmarks should be of
+// the same type. Exactly one stream to copy landmarks from, to copy to and to
+// output should be provided.
+//
+// Inputs:
+//   LANDMARKS_FROM (optional): A LandmarkList of landmarks to copy from.
+//   NORM_LANDMARKS_FROM (optional): A NormalizedLandmarkList of landmarks to
+//     copy from.
+//   LANDMARKS_TO (optional): A LandmarkList of landmarks to copy to.
+//   NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
+//     to.
+//
+// Outputs:
+//   LANDMARKS_TO (optional): A LandmarkList of landmarks from LANDMARKS_TO and
+//     visibility/presence from LANDMARKS_FROM or NORM_LANDMARKS_FROM.
+//   NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy
+//     to.
+//
+// Example config:
+//   node {
+//     calculator: "VisibilityCopyCalculator"
+//     input_stream: "NORM_LANDMARKS_FROM:pose_landmarks"
+//     input_stream: "LANDMARKS_TO:pose_world_landmarks"
+//     output_stream: "LANDMARKS_TO:pose_world_landmarks_with_visibility"
+//     options: {
+//       [mediapipe.VisibilityCopyCalculatorOptions.ext] {
+//         copy_visibility: true
+//         copy_presence: true
+//       }
+//     }
+//   }
+//
+class VisibilityCopyCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  template <class LandmarkFromType, class LandmarkToType>
+  absl::Status CopyVisibility(CalculatorContext* cc,
+                              const std::string& landmarks_from_tag,
+                              const std::string& landmarks_to_tag);
+
+  bool copy_visibility_;
+  bool copy_presence_;
+};
+REGISTER_CALCULATOR(VisibilityCopyCalculator);
+
+absl::Status VisibilityCopyCalculator::GetContract(CalculatorContract* cc) {
+  // Landmarks to copy from.
+  RET_CHECK(cc->Inputs().HasTag(kLandmarksFromTag) ^
+            cc->Inputs().HasTag(kNormalizedLandmarksFromTag))
+      << "Exatly one landmarks stream to copy from should be provided";
+  if (cc->Inputs().HasTag(kLandmarksFromTag)) {
+    cc->Inputs().Tag(kLandmarksFromTag).Set<LandmarkList>();
+  } else {
+    cc->Inputs().Tag(kNormalizedLandmarksFromTag).Set<NormalizedLandmarkList>();
+  }
+
+  // Landmarks to copy to and corresponding output landmarks.
+  RET_CHECK(cc->Inputs().HasTag(kLandmarksToTag) ^
+            cc->Inputs().HasTag(kNormalizedLandmarksToTag))
+      << "Exatly one landmarks stream to copy to should be provided";
+  if (cc->Inputs().HasTag(kLandmarksToTag)) {
+    cc->Inputs().Tag(kLandmarksToTag).Set<LandmarkList>();
+
+    RET_CHECK(cc->Outputs().HasTag(kLandmarksToTag))
+        << "Landmarks to copy to and output stream types should be the same";
+    cc->Outputs().Tag(kLandmarksToTag).Set<LandmarkList>();
+  } else {
+    cc->Inputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
+
+    RET_CHECK(cc->Outputs().HasTag(kNormalizedLandmarksToTag))
+        << "Landmarks to copy to and output stream types should be the same";
+    cc->Outputs().Tag(kNormalizedLandmarksToTag).Set<NormalizedLandmarkList>();
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status VisibilityCopyCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  const auto& options = cc->Options<VisibilityCopyCalculatorOptions>();
+  copy_visibility_ = options.copy_visibility();
+  copy_presence_ = options.copy_presence();
+
+  return absl::OkStatus();
+}
+
+absl::Status VisibilityCopyCalculator::Process(CalculatorContext* cc) {
+  // Switch between all four possible combinations of landmarks from and
+  // landmarks to types (normalized and non-normalized).
+  auto status = absl::OkStatus();
+  if (cc->Inputs().HasTag(kLandmarksFromTag)) {
+    if (cc->Inputs().HasTag(kLandmarksToTag)) {
+      status = CopyVisibility<LandmarkList, LandmarkList>(cc, kLandmarksFromTag,
+                                                          kLandmarksToTag);
+    } else {
+      status = CopyVisibility<LandmarkList, NormalizedLandmarkList>(
+          cc, kLandmarksFromTag, kNormalizedLandmarksToTag);
+    }
+  } else {
+    if (cc->Inputs().HasTag(kLandmarksToTag)) {
+      status = CopyVisibility<NormalizedLandmarkList, LandmarkList>(
+          cc, kNormalizedLandmarksFromTag, kLandmarksToTag);
+    } else {
+      status = CopyVisibility<NormalizedLandmarkList, NormalizedLandmarkList>(
+          cc, kNormalizedLandmarksFromTag, kNormalizedLandmarksToTag);
+    }
+  }
+
+  return status;
+}
+
+template <class LandmarkFromType, class LandmarkToType>
+absl::Status VisibilityCopyCalculator::CopyVisibility(
+    CalculatorContext* cc, const std::string& landmarks_from_tag,
+    const std::string& landmarks_to_tag) {
+  // Check that both landmarks to copy from and to copy to are non empty.
+  if (cc->Inputs().Tag(landmarks_from_tag).IsEmpty() ||
+      cc->Inputs().Tag(landmarks_to_tag).IsEmpty()) {
+    return absl::OkStatus();
+  }
+
+  const auto landmarks_from =
+      cc->Inputs().Tag(landmarks_from_tag).Get<LandmarkFromType>();
+  const auto landmarks_to =
+      cc->Inputs().Tag(landmarks_to_tag).Get<LandmarkToType>();
+  auto landmarks_out = absl::make_unique<LandmarkToType>();
+
+  for (int i = 0; i < landmarks_from.landmark_size(); ++i) {
+    const auto& landmark_from = landmarks_from.landmark(i);
+    const auto& landmark_to = landmarks_to.landmark(i);
+
+    // Create output landmark and copy all fields from the `to` landmark.
+    const auto& landmark_out = landmarks_out->add_landmark();
+    *landmark_out = landmark_to;
+
+    // Copy visibility and presence from the `from` landmark.
+    if (copy_visibility_) {
+      landmark_out->set_visibility(landmark_from.visibility());
+    }
+    if (copy_presence_) {
+      landmark_out->set_presence(landmark_from.presence());
+    }
+  }
+
+  cc->Outputs()
+      .Tag(landmarks_to_tag)
+      .Add(landmarks_out.release(), cc->InputTimestamp());
+
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/visibility_copy_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+
+message VisibilityCopyCalculatorOptions {
+  extend CalculatorOptions {
+    optional VisibilityCopyCalculatorOptions ext = 363728421;
+  }
+
+  optional bool copy_visibility = 1 [default = true];
+
+  optional bool copy_presence = 2 [default = true];
+}

mediapipe/calculators/util/visibility_smoothing_calculator.cc

@@ -0,0 +1,243 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+
+#include "absl/algorithm/container.h"
+#include "mediapipe/calculators/util/visibility_smoothing_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/util/filtering/low_pass_filter.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
+constexpr char kLandmarksTag[] = "LANDMARKS";
+constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
+constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
+
+using mediapipe::LowPassFilter;
+
+// Abstract class for various visibility filters.
+class VisibilityFilter {
+ public:
+  virtual ~VisibilityFilter() = default;
+
+  virtual absl::Status Reset() { return absl::OkStatus(); }
+
+  virtual absl::Status Apply(const LandmarkList& in_landmarks,
+                             const absl::Duration& timestamp,
+                             LandmarkList* out_landmarks) = 0;
+
+  virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+                             const absl::Duration& timestamp,
+                             NormalizedLandmarkList* out_landmarks) = 0;
+};
+
+// Returns visibility as is without smoothing.
+class NoFilter : public VisibilityFilter {
+ public:
+  absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     NormalizedLandmarkList* out_landmarks) override {
+    *out_landmarks = in_landmarks;
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     LandmarkList* out_landmarks) override {
+    *out_landmarks = in_landmarks;
+    return absl::OkStatus();
+  }
+};
+
+// Please check LowPassFilter documentation for details.
+class LowPassVisibilityFilter : public VisibilityFilter {
+ public:
+  LowPassVisibilityFilter(float alpha) : alpha_(alpha) {}
+
+  absl::Status Reset() override {
+    visibility_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     LandmarkList* out_landmarks) override {
+    return ApplyImpl<LandmarkList>(in_landmarks, timestamp, out_landmarks);
+  }
+
+  absl::Status Apply(const NormalizedLandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     NormalizedLandmarkList* out_landmarks) override {
+    return ApplyImpl<NormalizedLandmarkList>(in_landmarks, timestamp,
+                                             out_landmarks);
+  }
+
+ private:
+  template <class LandmarksType>
+  absl::Status ApplyImpl(const LandmarksType& in_landmarks,
+                         const absl::Duration& timestamp,
+                         LandmarksType* out_landmarks) {
+    // Initializes filters for the first time or after Reset. If initialized
+    // then check the size.
+    int n_landmarks = in_landmarks.landmark_size();
+    if (!visibility_filters_.empty()) {
+      RET_CHECK_EQ(visibility_filters_.size(), n_landmarks);
+    } else {
+      visibility_filters_.resize(n_landmarks, LowPassFilter(alpha_));
+    }
+
+    // Filter visibilities.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks->add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_visibility(
+          visibility_filters_[i].Apply(in_landmark.visibility()));
+    }
+
+    return absl::OkStatus();
+  }
+
+  float alpha_;
+  std::vector<LowPassFilter> visibility_filters_;
+};
+
+}  // namespace
+
+// A calculator to smooth landmark visibilities over time.
+//
+// Exactly one landmarks input stream is expected. Output stream type should be
+// the same as the input one.
+//
+// Inputs:
+//   LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
+//   NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
+//     to smooth.
+//
+// Outputs:
+//   FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
+//   NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
+//     landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "VisibilitySmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:pose_landmarks"
+//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
+//     options: {
+//       [mediapipe.VisibilitySmoothingCalculatorOptions.ext] {
+//         low_pass_filter: {
+//           alpha: 0.1
+//         }
+//       }
+//     }
+//   }
+//
+class VisibilitySmoothingCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  std::unique_ptr<VisibilityFilter> visibility_filter_;
+};
+REGISTER_CALCULATOR(VisibilitySmoothingCalculator);
+
+absl::Status VisibilitySmoothingCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag) ^
+            cc->Inputs().HasTag(kLandmarksTag))
+      << "Exactly one landmarks input stream is expected";
+  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
+    cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
+    RET_CHECK(cc->Outputs().HasTag(kNormalizedFilteredLandmarksTag))
+        << "Landmarks output stream should of the same type as input one";
+    cc->Outputs()
+        .Tag(kNormalizedFilteredLandmarksTag)
+        .Set<NormalizedLandmarkList>();
+  } else {
+    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
+    RET_CHECK(cc->Outputs().HasTag(kFilteredLandmarksTag))
+        << "Landmarks output stream should of the same type as input one";
+    cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status VisibilitySmoothingCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  // Pick visibility filter.
+  const auto& options = cc->Options<VisibilitySmoothingCalculatorOptions>();
+  if (options.has_no_filter()) {
+    visibility_filter_ = absl::make_unique<NoFilter>();
+  } else if (options.has_low_pass_filter()) {
+    visibility_filter_ = absl::make_unique<LowPassVisibilityFilter>(
+        options.low_pass_filter().alpha());
+  } else {
+    RET_CHECK_FAIL()
+        << "Visibility filter is either not specified or not supported";
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status VisibilitySmoothingCalculator::Process(CalculatorContext* cc) {
+  // Check that landmarks are not empty and reset the filter if so.
+  // Don't emit an empty packet for this timestamp.
+  if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
+       cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
+      (cc->Inputs().HasTag(kLandmarksTag) &&
+       cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
+    MP_RETURN_IF_ERROR(visibility_filter_->Reset());
+    return absl::OkStatus();
+  }
+
+  const auto& timestamp =
+      absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
+    const auto& in_landmarks =
+        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
+    auto out_landmarks = absl::make_unique<NormalizedLandmarkList>();
+    MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
+                                                 out_landmarks.get()));
+    cc->Outputs()
+        .Tag(kNormalizedFilteredLandmarksTag)
+        .Add(out_landmarks.release(), cc->InputTimestamp());
+  } else {
+    const auto& in_landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
+    auto out_landmarks = absl::make_unique<LandmarkList>();
+    MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp,
+                                                 out_landmarks.get()));
+    cc->Outputs()
+        .Tag(kFilteredLandmarksTag)
+        .Add(out_landmarks.release(), cc->InputTimestamp());
+  }
+
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/visibility_smoothing_calculator.proto

@@ -0,0 +1,40 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+
+message VisibilitySmoothingCalculatorOptions {
+  extend CalculatorOptions {
+    optional VisibilitySmoothingCalculatorOptions ext = 360207350;
+  }
+
+  // Default behaviour and fast way to disable smoothing.
+  message NoFilter {}
+
+  message LowPassFilter {
+    // Coefficient applied to a new value, whilte `1 - alpha` is applied to a
+    // stored value. Should be in [0, 1] range. The smaller the value - the
+    // smoother result and the bigger lag.
+    optional float alpha = 1 [default = 0.1];
+  }
+
+  oneof filter_options {
+    NoFilter no_filter = 1;
+    LowPassFilter low_pass_filter = 2;
+  }
+}

mediapipe/calculators/util/world_landmark_projection_calculator.cc

@@ -0,0 +1,108 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kLandmarksTag[] = "LANDMARKS";
+constexpr char kRectTag[] = "NORM_RECT";
+
+}  // namespace
+
+// Projects world landmarks from the rectangle to original coordinates.
+//
+// World landmarks are predicted in meters rather than in pixels of the image
+// and have origin in the middle of the hips rather than in the corner of the
+// pose image (cropped with given rectangle). Thus only rotation (but not scale
+// and translation) is applied to the landmarks to transform them back to
+// original coordinates.
+//
+// Input:
+//   LANDMARKS: A LandmarkList representing world landmarks in the rectangle.
+//   NORM_RECT: An NormalizedRect representing a normalized rectangle in image
+//              coordinates.
+//
+// Output:
+//   LANDMARKS: A LandmarkList representing world landmarks projected (rotated
+//              but not scaled or translated) from the rectangle to original
+//              coordinates.
+//
+// Usage example:
+// node {
+//   calculator: "WorldLandmarkProjectionCalculator"
+//   input_stream: "LANDMARKS:landmarks"
+//   input_stream: "NORM_RECT:rect"
+//   output_stream: "LANDMARKS:projected_landmarks"
+// }
+//
+class WorldLandmarkProjectionCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
+    cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
+    cc->Outputs().Tag(kLandmarksTag).Set<LandmarkList>();
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks and rect are not empty.
+    if (cc->Inputs().Tag(kLandmarksTag).IsEmpty() ||
+        cc->Inputs().Tag(kRectTag).IsEmpty()) {
+      return absl::OkStatus();
+    }
+
+    const auto& in_landmarks =
+        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
+    const auto& in_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
+
+    auto out_landmarks = absl::make_unique<LandmarkList>();
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      Landmark* out_landmark = out_landmarks->add_landmark();
+      *out_landmark = in_landmark;
+
+      const float angle = in_rect.rotation();
+      out_landmark->set_x(std::cos(angle) * in_landmark.x() -
+                          std::sin(angle) * in_landmark.y());
+      out_landmark->set_y(std::sin(angle) * in_landmark.x() +
+                          std::cos(angle) * in_landmark.y());
+    }
+
+    cc->Outputs()
+        .Tag(kLandmarksTag)
+        .Add(out_landmarks.release(), cc->InputTimestamp());
+
+    return absl::OkStatus();
+  }
+};
+REGISTER_CALCULATOR(WorldLandmarkProjectionCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/video/BUILD

@@ -426,6 +426,7 @@ cc_test(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -450,6 +451,7 @@ cc_test(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:opencv_video",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -534,6 +536,7 @@ cc_test(
         "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
         "//mediapipe/util/tracking:box_tracker_cc_proto",
         "//mediapipe/util/tracking:tracking_cc_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 

mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc

@@ -31,11 +31,11 @@ namespace {
 
 TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvVideoDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "VIDEO:video"
-        output_stream: "VIDEO_PRESTREAM:video_prestream")");
+        output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -77,11 +77,11 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) {
 
 TEST(OpenCvVideoDecoderCalculatorTest, TestFlvH264Video) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvVideoDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "VIDEO:video"
-        output_stream: "VIDEO_PRESTREAM:video_prestream")");
+        output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -121,11 +121,11 @@ TEST(OpenCvVideoDecoderCalculatorTest, TestFlvH264Video) {
 
 TEST(OpenCvVideoDecoderCalculatorTest, TestMkvVp8Video) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvVideoDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "VIDEO:video"
-        output_stream: "VIDEO_PRESTREAM:video_prestream")");
+        output_stream: "VIDEO_PRESTREAM:video_prestream")pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",

mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc

@@ -36,26 +36,28 @@ namespace {
 // TODO: Investigate the “Could not open codec 'libx264'” error with
 // opencv2.
 TEST(OpenCvVideoEncoderCalculatorTest, DISABLED_TestMp4Avc720pVideo) {
-  CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  CalculatorGraphConfig config =
-    node {
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
-      calculator: "OpenCvVideoDecoderCalculator"
+        node {
-      input_side_packet: "INPUT_FILE_PATH:input_file_path"
+          calculator: "OpenCvVideoDecoderCalculator"
-      output_stream: "VIDEO:video"
+          input_side_packet: "INPUT_FILE_PATH:input_file_path"
-      output_stream: "VIDEO_PRESTREAM:video_prestream"
+          output_stream: "VIDEO:video"
-    }
+          output_stream: "VIDEO_PRESTREAM:video_prestream"
-    node {
-      calculator: "OpenCvVideoEncoderCalculator"
-      input_stream: "VIDEO:video"
-      input_stream: "VIDEO_PRESTREAM:video_prestream"
-      input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
-      node_options {
-        [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
-          codec: "avc1"
-          video_format: "mp4"
         }
-      }
+        node {
-    }
+          calculator: "OpenCvVideoEncoderCalculator"
-  )");
+          input_stream: "VIDEO:video"
+          input_stream: "VIDEO_PRESTREAM:video_prestream"
+          input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
+          node_options {
+            [type.googleapis.com/
+             mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+              codec: "avc1"
+              video_format: "mp4"
+            }
+          }
+        }
+      )pb");
   std::map<std::string, Packet> input_side_packets;
   input_side_packets["input_file_path"] = MakePacket<std::string>(
       file::JoinPath("./",
@@ -95,26 +97,28 @@ TEST(OpenCvVideoEncoderCalculatorTest, DISABLED_TestMp4Avc720pVideo) {
 }
 
 TEST(OpenCvVideoEncoderCalculatorTest, TestFlvH264Video) {
-  CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  CalculatorGraphConfig config =
-    node {
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
-      calculator: "OpenCvVideoDecoderCalculator"
+        node {
-      input_side_packet: "INPUT_FILE_PATH:input_file_path"
+          calculator: "OpenCvVideoDecoderCalculator"
-      output_stream: "VIDEO:video"
+          input_side_packet: "INPUT_FILE_PATH:input_file_path"
-      output_stream: "VIDEO_PRESTREAM:video_prestream"
+          output_stream: "VIDEO:video"
-    }
+          output_stream: "VIDEO_PRESTREAM:video_prestream"
-    node {
-      calculator: "OpenCvVideoEncoderCalculator"
-      input_stream: "VIDEO:video"
-      input_stream: "VIDEO_PRESTREAM:video_prestream"
-      input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
-      node_options {
-        [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
-          codec: "MJPG"
-          video_format: "avi"
         }
-      }
+        node {
-    }
+          calculator: "OpenCvVideoEncoderCalculator"
-  )");
+          input_stream: "VIDEO:video"
+          input_stream: "VIDEO_PRESTREAM:video_prestream"
+          input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
+          node_options {
+            [type.googleapis.com/
+             mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+              codec: "MJPG"
+              video_format: "avi"
+            }
+          }
+        }
+      )pb");
   std::map<std::string, Packet> input_side_packets;
   input_side_packets["input_file_path"] = MakePacket<std::string>(
       file::JoinPath("./",
@@ -156,26 +160,28 @@ TEST(OpenCvVideoEncoderCalculatorTest, TestFlvH264Video) {
 }
 
 TEST(OpenCvVideoEncoderCalculatorTest, TestMkvVp8Video) {
-  CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  CalculatorGraphConfig config =
-    node {
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
-      calculator: "OpenCvVideoDecoderCalculator"
+        node {
-      input_side_packet: "INPUT_FILE_PATH:input_file_path"
+          calculator: "OpenCvVideoDecoderCalculator"
-      output_stream: "VIDEO:video"
+          input_side_packet: "INPUT_FILE_PATH:input_file_path"
-      output_stream: "VIDEO_PRESTREAM:video_prestream"
+          output_stream: "VIDEO:video"
-    }
+          output_stream: "VIDEO_PRESTREAM:video_prestream"
-    node {
-      calculator: "OpenCvVideoEncoderCalculator"
-      input_stream: "VIDEO:video"
-      input_stream: "VIDEO_PRESTREAM:video_prestream"
-      input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
-      node_options {
-        [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
-          codec: "PIM1"
-          video_format: "mkv"
         }
-      }
+        node {
-    }
+          calculator: "OpenCvVideoEncoderCalculator"
-  )");
+          input_stream: "VIDEO:video"
+          input_stream: "VIDEO_PRESTREAM:video_prestream"
+          input_side_packet: "OUTPUT_FILE_PATH:output_file_path"
+          node_options {
+            [type.googleapis.com/
+             mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+              codec: "PIM1"
+              video_format: "mkv"
+            }
+          }
+        }
+      )pb");
   std::map<std::string, Packet> input_side_packets;
   input_side_packets["input_file_path"] = MakePacket<std::string>(
       file::JoinPath("./",

mediapipe/calculators/video/video_pre_stream_calculator_test.cc

@@ -25,7 +25,7 @@ namespace mediapipe {
 namespace {
 
 TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) {
-  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
     input_stream: "input"
     node {
       calculator: "VideoPreStreamCalculator"
@@ -34,7 +34,7 @@ TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) {
       options {
         [mediapipe.VideoPreStreamCalculatorOptions.ext] { fps { value: 3 } }
       }
-    })");
+    })pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(config));
   auto poller_status = graph.AddOutputStreamPoller("output");
@@ -66,7 +66,7 @@ TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) {
 }
 
 TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) {
-  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
     input_stream: "frame"
     input_stream: "input_header"
     node {
@@ -74,7 +74,7 @@ TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) {
       input_stream: "FRAME:frame"
       input_stream: "VIDEO_PRESTREAM:input_header"
       output_stream: "output_header"
-    })");
+    })pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(config));
   auto poller_status = graph.AddOutputStreamPoller("output_header");
@@ -104,13 +104,13 @@ TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) {
 }
 
 TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInOptions) {
-  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
     input_stream: "frame"
     node {
       calculator: "VideoPreStreamCalculator"
       input_stream: "frame"
       output_stream: "output_header"
-    })");
+    })pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -126,7 +126,7 @@ TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInOptions) {
 
 // Input header missing.
 TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) {
-  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
     input_stream: "frame"
     input_stream: "input_header"
     node {
@@ -135,7 +135,7 @@ TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) {
       input_stream: "VIDEO_PRESTREAM:input_header"
       output_stream: "output_header"
     }
-  )");
+  )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -152,7 +152,7 @@ TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) {
 
 // Input header not at prestream (before, with, and after frame data).
 TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream2) {
-  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
     input_stream: "frame"
     input_stream: "input_header"
     node {
@@ -161,7 +161,7 @@ TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream2) {
       input_stream: "VIDEO_PRESTREAM:input_header"
       output_stream: "output_header"
     }
-  )");
+  )pb");
 
   for (int64 timestamp = -1; timestamp < 2; ++timestamp) {
     CalculatorGraph graph;