Project import generated by Copybara.

GitOrigin-RevId: 1e221238b0bc717115c8152ad3092da3309a63a1

.github/ISSUE_TEMPLATE/00-build-installation-issue.md

@@ -0,0 +1,21 @@
+<em>Please make sure that this is a build/installation issue and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html) documentation before raising any issues.</em>
+
+**System information** (Please provide as much relevant information as possible)
+- OS Platform and Distribution (e.g. Linux Ubuntu 16.04, Android 11, iOS 14.4):
+- Compiler version (e.g. gcc/g++ 8 /Apple clang version 12.0.0):
+- Programming Language and version ( e.g. C++ 14, Python 3.6, Java ):
+- Installed using virtualenv? pip? Conda? (if python):
+- [MediaPipe version](https://github.com/google/mediapipe/releases):
+- Bazel version:
+- XCode and Tulsi versions (if iOS):
+- Android SDK and NDK versions (if android):
+- Android [AAR](https://google.github.io/mediapipe/getting_started/android_archive_library.html) ( if android):
+- OpenCV version (if running on desktop):
+
+**Describe the problem**:
+
+
+**[Provide the exact sequence of commands / steps that you executed before running into the problem](https://google.github.io/mediapipe/getting_started/getting_started.html):**
+
+**Complete Logs:**
+Include Complete Log information or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached:

.github/ISSUE_TEMPLATE/10-solution-issue.md

@@ -0,0 +1,20 @@
+<em>Please make sure that this is a [solution](https://google.github.io/mediapipe/solutions/solutions.html) issue.<em>
+
+**System information** (Please provide as much relevant information as possible)
+- Have I written custom code (as opposed to using a stock example script provided in Mediapipe):
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
+- [MediaPipe version](https://github.com/google/mediapipe/releases):
+- Bazel version:
+- Solution (e.g. FaceMesh, Pose, Holistic):
+- Programming Language and version ( e.g. C++, Python, Java):
+
+**Describe the expected behavior:**
+
+**Standalone code you may have used to try to get what you need :**
+
+If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab/repo link /any notebook:
+
+**Other info / Complete Logs :**
+Include any logs or source code that would be helpful to
+diagnose the problem. If including tracebacks, please include the full
+traceback. Large logs and files should be attached:

.github/ISSUE_TEMPLATE/20-documentation-issue.md

@@ -0,0 +1,45 @@
+Thank you for submitting a MediaPipe documentation issue.
+The MediaPipe docs are open source! To get involved, read the documentation Contributor Guide
+## URL(s) with the issue:
+
+Please provide a link to the documentation entry, for example: https://github.com/google/mediapipe/blob/master/docs/solutions/face_mesh.md#models
+
+## Description of issue (what needs changing):
+
+Kinds of documentation problems:
+
+### Clear description
+
+For example, why should someone use this method? How is it useful?
+
+### Correct links
+
+Is the link to the source code correct?
+
+### Parameters defined
+Are all parameters defined and formatted correctly?
+
+### Returns defined
+
+Are return values defined?
+
+### Raises listed and defined
+
+Are the errors defined? For example,
+
+### Usage example
+
+Is there a usage example?
+
+See the API guide:
+on how to write testable usage examples.
+
+### Request visuals, if applicable
+
+Are there currently visuals? If not, will it clarify the content?
+
+### Submit a pull request?
+
+Are you planning to also submit a pull request to fix the issue? See the docs
+https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md
+

.github/bot_config.yml

@@ -0,0 +1,18 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+# A list of assignees
+assignees:
+   - sgowroji

.github/stale.yml

@@ -0,0 +1,34 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+#
+# This file was assembled from multiple pieces, whose use is documented
+# throughout. Please refer to the TensorFlow dockerfiles documentation
+# for more information.
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 7
+# Number of days of inactivity before a stale Issue or Pull Request is closed
+daysUntilClose: 7
+# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
+onlyLabels:
+ - stat:awaiting response
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you.
+# Comment to post when removing the stale label. Set to `false` to disable
+unmarkComment: false
+closeComment: >
+  Closing as stale. Please reopen if you'd like to work on this further.

Dockerfile

@@ -23,6 +23,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
+        gcc-8 g++-8 \
         ca-certificates \
         curl \
         ffmpeg \
@@ -44,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100 --slave /usr/bin/g++ g++ /usr/bin/g++-8
 RUN pip3 install --upgrade setuptools
 RUN pip3 install wheel
 RUN pip3 install future
@@ -54,7 +56,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.4.1
+ARG BAZEL_VERSION=3.7.2
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

MANIFEST.in

@@ -10,3 +10,10 @@ include requirements.txt
 recursive-include mediapipe/modules *.tflite *.txt *.binarypb
 exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
 exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite
+exclude mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite
+exclude mediapipe/modules/pose_landmark/pose_landmark_lite.tflite
+exclude mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite

README.md

@@ -40,11 +40,12 @@ Hair Segmentation
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Pose](https://google.github.io/mediapipe/solutions/pose)                                       | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Holistic](https://google.github.io/mediapipe/solutions/holistic)                               | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
+[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation)         | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)             | ✅                                                               |                                                         | ✅                                                       |                                                               |                                                               |
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

WORKSPACE

@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
-    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+    urls = [
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+    ],
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
 )
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "3.4.0")
-
+versions.check(minimum_bazel_version = "3.7.2")
 
 # ABSL cpp library lts_2020_09_23
 http_archive(
@@ -38,8 +41,8 @@ http_archive(
 
 http_archive(
    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-main",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/main.zip",
+   strip_prefix = "rules_foreign_cc-0.1.0",
+   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -68,8 +71,8 @@ http_archive(
 # Google Benchmark library.
 http_archive(
     name = "com_google_benchmark",
-    urls = ["https://github.com/google/benchmark/archive/master.zip"],
-    strip_prefix = "benchmark-master",
+    urls = ["https://github.com/google/benchmark/archive/main.zip"],
+    strip_prefix = "benchmark-main",
     build_file = "@//third_party:benchmark.BUILD",
 )
 
@@ -117,7 +120,8 @@ http_archive(
 # libyuv
 http_archive(
     name = "libyuv",
-    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
+    # Error: operand type mismatch for `vbroadcastss' caused by commit 8a13626e42f7fdcf3a6acbb0316760ee54cda7d8.
+    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/2525698acba9bf9b701ba6b4d9584291a1f62257.tar.gz"],
     build_file = "@//third_party:libyuv.BUILD",
 )
 
@@ -304,8 +308,8 @@ http_archive(
 
 # Maven dependencies.
 
-RULES_JVM_EXTERNAL_TAG = "3.2"
-RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+RULES_JVM_EXTERNAL_TAG = "4.0"
+RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
 
 http_archive(
     name = "rules_jvm_external",
@@ -318,7 +322,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 
 # Important: there can only be one maven_install rule. Add new maven deps here.
 maven_install(
-    name = "maven",
     artifacts = [
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.2.0",
@@ -334,6 +337,8 @@ maven_install(
         "androidx.test.espresso:espresso-core:3.1.1",
         "com.github.bumptech.glide:glide:4.11.0",
         "com.google.android.material:material:aar:1.0.0-rc01",
+        "com.google.auto.value:auto-value:1.6.4",
+        "com.google.auto.value:auto-value-annotations:1.6.4",
         "com.google.code.findbugs:jsr305:3.0.2",
         "com.google.flogger:flogger-system-backend:0.3.1",
         "com.google.flogger:flogger:0.3.1",
@@ -343,10 +348,10 @@ maven_install(
         "org.hamcrest:hamcrest-library:1.3",
     ],
     repositories = [
-        "https://jcenter.bintray.com",
         "https://maven.google.com",
         "https://dl.google.com/dl/android/maven2",
         "https://repo1.maven.org/maven2",
+        "https://jcenter.bintray.com",
     ],
     fetch_sources = True,
     version_conflict_policy = "pinned",
@@ -363,10 +368,10 @@ http_archive(
     ],
 )
 
-#Tensorflow repo should always go after the other external dependencies.
-# 2020-12-09
-_TENSORFLOW_GIT_COMMIT = "0eadbb13cef1226b1bae17c941f7870734d97f8a"
-_TENSORFLOW_SHA256= "4ae06daa5b09c62f31b7bc1f781fd59053f286dd64355830d8c2ac601b795ef0"
+# Tensorflow repo should always go after the other external dependencies.
+# 2021-05-27
+_TENSORFLOW_GIT_COMMIT = "d6bfcdb0926173dbb7aa02ceba5aae6250b8aaa6"
+_TENSORFLOW_SHA256 = "ec40e1462239d8783d02f76a43412c8f80bac71ea20e41e1b7729b990aad6923"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -383,5 +388,7 @@ http_archive(
     sha256 = _TENSORFLOW_SHA256,
 )
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace(tf_repo_name = "org_tensorflow")
+load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
+tf_workspace3()
+load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+tf_workspace2()

build_desktop_examples.sh

@@ -17,15 +17,15 @@
 # Script to build/run all MediaPipe desktop example apps (with webcam input).
 #
 # To build and run all apps and store them in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir
+#   $ ./build_desktop_examples.sh -d out_dir
 #   Omitting -d and the associated directory saves all generated apps in the
 #   current directory.
 # To build all apps and store them in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir -b
+#   $ ./build_desktop_examples.sh -d out_dir -b
 #   Omitting -d and the associated directory saves all generated apps in the
 #   current directory.
 # To run all apps already stored in out_dir:
-#   $ ./build_ios_examples.sh -d out_dir -r
+#   $ ./build_desktop_examples.sh -d out_dir -r
 #   Omitting -d and the associated directory assumes all apps are in the current
 #   directory.
 
@@ -97,6 +97,7 @@ for app in ${apps}; do
       if [[ ${target_name} == "holistic_tracking" ||
             ${target_name} == "iris_tracking" ||
             ${target_name} == "pose_tracking" ||
+            ${target_name} == "selfie_segmentation" ||
             ${target_name} == "upper_body_pose_tracking" ]]; then
         graph_suffix="cpu"
       else

docs/framework_concepts/calculators.md

@@ -187,7 +187,7 @@ node {
 ```
 
 In the calculator implementation, inputs and outputs are also identified by tag
-name and index number. In the function below input are output are identified:
+name and index number. In the function below input and output are identified:
 
 *   By index number: The combined input stream is identified simply by index
     `0`.
@@ -248,12 +248,58 @@ absl::Status MyCalculator::Process() {
 }
 ```
 
+## Calculator options
+
+Calculators accept processing parameters through (1) input stream packets (2)
+input side packets, and (3) calculator options. Calculator options, if
+specified, appear as literal values in the `node_options` field of the
+`CalculatorGraphConfiguration.Node` message.
+
+```
+  node {
+    calculator: "TfLiteInferenceCalculator"
+    input_stream: "TENSORS:main_model_input"
+    output_stream: "TENSORS:main_model_output"
+    node_options: {
+      [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+        model_path: "mediapipe/models/active_speaker_detection/audio_visual_model.tflite"
+      }
+    }
+  }
+```
+
+The `node_options` field accepts the proto3 syntax.  Alternatively, calculator
+options can be specified in the `options` field using proto2 syntax.
+
+```
+  node: {
+    calculator: "IntervalFilterCalculator"
+    node_options: {
+      [type.googleapis.com/mediapipe.IntervalFilterCalculatorOptions] {
+        intervals {
+          start_us: 20000
+          end_us: 40000
+        }
+      }
+    }
+  }
+```
+
+Not all calculators accept calcuator options. In order to accept options, a
+calculator will normally define a new protobuf message type to represent its
+options, such as `IntervalFilterCalculatorOptions`. The calculator will then
+read that protobuf message in its `CalculatorBase::Open` method, and possibly
+also in the `CalculatorBase::GetContract` function or its
+`CalculatorBase::Process` method. Normally, the new protobuf message type will
+be defined as a protobuf schema using a ".proto" file and a
+`mediapipe_proto_library()` build rule.
+
 ## Example calculator
 
 This section discusses the implementation of `PacketClonerCalculator`, which
 does a relatively simple job, and is used in many calculator graphs.
-`PacketClonerCalculator` simply produces a copy of its most recent input
-packets on demand.
+`PacketClonerCalculator` simply produces a copy of its most recent input packets
+on demand.
 
 `PacketClonerCalculator` is useful when the timestamps of arriving data packets
 are not aligned perfectly. Suppose we have a room with a microphone, light
@@ -279,8 +325,8 @@ input streams:
     imageframe of video data representing video collected from camera in the
     room with timestamp.
 
-Below is the implementation of the `PacketClonerCalculator`.  You can see
-the `GetContract()`, `Open()`, and `Process()` methods as well as the instance
+Below is the implementation of the `PacketClonerCalculator`. You can see the
+`GetContract()`, `Open()`, and `Process()` methods as well as the instance
 variable `current_` which holds the most recent input packets.
 
 ```c++
@@ -355,7 +401,6 @@ class PacketClonerCalculator : public CalculatorBase {
               current_[i].At(cc->InputTimestamp()));
           // Add a packet to output stream of index i a packet from inputstream i
           // with timestamp common to all present inputs
-          //
         } else {
           cc->Outputs().Index(i).SetNextTimestampBound(
               cc->InputTimestamp().NextAllowedInStream());
@@ -382,7 +427,7 @@ defined your calculator class, register it with a macro invocation
 REGISTER_CALCULATOR(calculator_class_name).
 
 Below is a trivial MediaPipe graph that has 3 input streams, 1 node
-(PacketClonerCalculator) and 3 output streams.
+(PacketClonerCalculator) and 2 output streams.
 
 ```proto
 input_stream: "room_mic_signal"
@@ -402,6 +447,6 @@ node {
 The diagram below shows how the `PacketClonerCalculator` defines its output
 packets (bottom) based on its series of input packets (top).
 
-| ![Graph using PacketClonerCalculator](../images/packet_cloner_calculator.png) |
-| :---------------------------------------------------------------------------: |
-| *Each time it receives a packet on its TICK input stream, the PacketClonerCalculator outputs the most recent packet from each of its input streams. The sequence of output packets (bottom) is determined by the sequence of input packets (top) and their timestamps. The timestamps are shown along the right side of the diagram.* |
+![Graph using PacketClonerCalculator](../images/packet_cloner_calculator.png)  |
+:--------------------------------------------------------------------------: |
+*Each time it receives a packet on its TICK input stream, the PacketClonerCalculator outputs the most recent packet from each of its input streams. The sequence of output packets (bottom) is determined by the sequence of input packets (top) and their timestamps. The timestamps are shown along the right side of the diagram.* |

docs/framework_concepts/framework_concepts.md

@@ -110,3 +110,12 @@ Other policies are also available, implemented using a separate kind of
 component known as an InputStreamHandler.
 
 See [Synchronization](synchronization.md) for more details.
+
+### Real-time streams
+
+MediaPipe calculator graphs are often used to process streams of video or audio
+frames for interactive applications. Normally, each Calculator runs as soon as
+all of its input packets for a given timestamp become available. Calculators
+used in real-time graphs need to define output timestamp bounds based on input
+timestamp bounds in order to allow downstream calculators to be scheduled
+promptly. See [Real-time Streams](realtime_streams.md) for details.

docs/framework_concepts/graphs.md

@@ -83,12 +83,12 @@ Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
     output_stream: "out3"
 
     node {
-        calculator: "PassThroughculator"
+        calculator: "PassThroughCalculator"
         input_stream: "out1"
         output_stream: "out2"
     }
     node {
-        calculator: "PassThroughculator"
+        calculator: "PassThroughCalculator"
         input_stream: "out2"
         output_stream: "out3"
     }

docs/framework_concepts/packets.md

@@ -12,19 +12,30 @@ nav_order: 3
 {:toc}
 ---
 
-Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
+Calculators communicate by sending and receiving packets. Typically a single
+packet is sent along each input stream at each input timestamp. A packet can
+contain any kind of data, such as a single frame of video or a single integer
+detection count.
 
 ## Creating a packet
 
-Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
+Packets are generally created with `mediapipe::MakePacket<T>()` or
+`mediapipe::Adopt()` (from packet.h).
 
 ```c++
-// Create some data.
-auto data = absl::make_unique<MyDataClass>("constructor_argument");
-// Create a packet to own the data.
-Packet p = Adopt(data.release());
+// Create a packet containing some new data.
+Packet p = MakePacket<MyDataClass>("constructor_argument");
 // Make a new packet with the same data and a different timestamp.
 Packet p2 = p.At(Timestamp::PostStream());
 ```
 
+or:
+
+```c++
+// Create some new data.
+auto data = absl::make_unique<MyDataClass>("constructor_argument");
+// Create a packet to own the data.
+Packet p = Adopt(data.release()).At(Timestamp::PostStream());
+```
+
 Data within a packet is accessed with `Packet::Get<T>()`

docs/framework_concepts/realtime_streams.md

@@ -0,0 +1,186 @@
+---
+layout: default
+title: Real-time Streams
+parent: Framework Concepts
+nav_order: 6
+---
+
+# Real-time Streams
+{: .no_toc }
+
+1. TOC
+{:toc}
+---
+
+## Real-time timestamps
+
+MediaPipe calculator graphs are often used to process streams of video or audio
+frames for interactive applications. The MediaPipe framework requires only that
+successive packets be assigned monotonically increasing timestamps. By
+convention, real-time calculators and graphs use the recording time or the
+presentation time of each frame as its timestamp, with each timestamp indicating
+the microseconds since `Jan/1/1970:00:00:00`. This allows packets from various
+sources to be processed in a globally consistent sequence.
+
+## Real-time scheduling
+
+Normally, each Calculator runs as soon as all of its input packets for a given
+timestamp become available. Normally, this happens when the calculator has
+finished processing the previous frame, and each of the calculators producing
+its inputs have finished processing the current frame. The MediaPipe scheduler
+invokes each calculator as soon as these conditions are met. See
+[Synchronization](synchronization.md) for more details.
+
+## Timestamp bounds
+
+When a calculator does not produce any output packets for a given timestamp, it
+can instead output a "timestamp bound" indicating that no packet will be
+produced for that timestamp. This indication is necessary to allow downstream
+calculators to run at that timestamp, even though no packet has arrived for
+certain streams for that timestamp. This is especially important for real-time
+graphs in interactive applications, where it is crucial that each calculator
+begin processing as soon as possible.
+
+Consider a graph like the following:
+
+```
+node {
+   calculator: "A"
+   input_stream: "alpha_in"
+   output_stream: "alpha"
+}
+node {
+   calculator: "B"
+   input_stream: "alpha"
+   input_stream: "foo"
+   output_stream: "beta"
+}
+```
+
+Suppose: at timestamp `T`, node `A` doesn't send a packet in its output stream
+`alpha`. Node `B` gets a packet in `foo` at timestamp `T` and is waiting for a
+packet in `alpha` at timestamp `T`. If `A` doesn't send `B` a timestamp bound
+update for `alpha`, `B` will keep waiting for a packet to arrive in `alpha`.
+Meanwhile, the packet queue of `foo` will accumulate packets at `T`, `T+1` and
+so on.
+
+To output a packet on a stream, a calculator uses the API functions
+`CalculatorContext::Outputs` and `OutputStream::Add`. To instead output a
+timestamp bound on a stream, a calculator can use the API functions
+`CalculatorContext::Outputs` and `CalculatorContext::SetNextTimestampBound`. The
+specified bound is the lowest allowable timestamp for the next packet on the
+specified output stream. When no packet is output, a calculator will typically
+do something like:
+
+```
+cc->Outputs().Tag("output_frame").SetNextTimestampBound(
+  cc->InputTimestamp().NextAllowedInStream());
+```
+
+The function `Timestamp::NextAllowedInStream` returns the successive timestamp.
+For example, `Timestamp(1).NextAllowedInStream() == Timestamp(2)`.
+
+## Propagating timestamp bounds
+
+Calculators that will be used in real-time graphs need to define output
+timestamp bounds based on input timestamp bounds in order to allow downstream
+calculators to be scheduled promptly. A common pattern is for calculators to
+output packets with the same timestamps as their input packets. In this case,
+simply outputting a packet on every call to `Calculator::Process` is sufficient
+to define output timestamp bounds.
+
+However, calculators are not required to follow this common pattern for output
+timestamps, they are only required to choose monotonically increasing output
+timestamps. As a result, certain calculators must calculate timestamp bounds
+explicitly. MediaPipe provides several tools for computing appropriate timestamp
+bound for each calculator.
+
+1\. **SetNextTimestampBound()** can be used to specify the timestamp bound, `t +
+1`, for an output stream.
+
+```
+cc->Outputs.Tag("OUT").SetNextTimestampBound(t.NextAllowedInStream());
+```
+
+Alternatively, an empty packet with timestamp `t` can be produced to specify the
+timestamp bound `t + 1`.
+
+```
+cc->Outputs.Tag("OUT").Add(Packet(), t);
+```
+
+The timestamp bound of an input stream is indicated by the packet or the empty
+packet on the input stream.
+
+```
+Timestamp bound = cc->Inputs().Tag("IN").Value().Timestamp();
+```
+
+2\. **TimestampOffset()** can be specified in order to automatically copy the
+timestamp bound from input streams to output streams.
+
+```
+cc->SetTimestampOffset(0);
+```
+
+This setting has the advantage of propagating timestamp bounds automatically,
+even when only timestamp bounds arrive and Calculator::Process is not invoked.
+
+3\. **ProcessTimestampBounds()** can be specified in order to invoke
+`Calculator::Process` for each new "settled timestamp", where the "settled
+timestamp" is the new highest timestamp below the current timestamp bounds.
+Without `ProcessTimestampBounds()`, `Calculator::Process` is invoked only with
+one or more arriving packets.
+
+```
+cc->SetProcessTimestampBounds(true);
+```
+
+This setting allows a calculator to perform its own timestamp bounds calculation
+and propagation, even when only input timestamps are updated. It can be used to
+replicate the effect of `TimestampOffset()`, but it can also be used to
+calculate a timestamp bound that takes into account additional factors.
+
+For example, in order to replicate `SetTimestampOffset(0)`, a calculator could
+do the following:
+
+```
+absl::Status Open(CalculatorContext* cc) {
+  cc->SetProcessTimestampBounds(true);
+}
+
+absl::Status Process(CalculatorContext* cc) {
+  cc->Outputs.Tag("OUT").SetNextTimestampBound(
+      cc->InputTimestamp().NextAllowedInStream());
+}
+```
+
+## Scheduling of Calculator::Open and Calculator::Close
+
+`Calculator::Open` is invoked when all required input side-packets have been
+produced. Input side-packets can be provided by the enclosing application or by
+"side-packet calculators" inside the graph. Side-packets can be specified from
+outside the graph using the API's `CalculatorGraph::Initialize` and
+`CalculatorGraph::StartRun`. Side packets can be specified by calculators within
+the graph using `CalculatorGraphConfig::OutputSidePackets` and
+`OutputSidePacket::Set`.
+
+Calculator::Close is invoked when all of the input streams have become `Done` by
+being closed or reaching timestamp bound `Timestamp::Done`.
+
+**Note:** If the graph finishes all pending calculator execution and becomes
+`Done`, before some streams become `Done`, then MediaPipe will invoke the
+remaining calls to `Calculator::Close`, so that every calculator can produce its
+final outputs.
+
+The use of `TimestampOffset` has some implications for `Calculator::Close`. A
+calculator specifying `SetTimestampOffset(0)` will by design signal that all of
+its output streams have reached `Timestamp::Done` when all of its input streams
+have reached `Timestamp::Done`, and therefore no further outputs are possible.
+This prevents such a calculator from emitting any packets during
+`Calculator::Close`. If a calculator needs to produce a summary packet during
+`Calculator::Close`, `Calculator::Process` must specify timestamp bounds such
+that at least one timestamp (such as `Timestamp::Max`) remains available during
+`Calculator::Close`. This means that such a calculator normally cannot rely upon
+`SetTimestampOffset(0)` and must instead specify timestamp bounds explicitly
+using `SetNextTimestampBounds()`.

docs/getting_started/android.md

@@ -28,7 +28,7 @@ Gradle.
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
 *   Setup Android SDK release 28.0.3 and above.
-*   Setup Android NDK r18b and above.
+*   Setup Android NDK version between 18 and 21.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
 below for Android Studio setup). However, if you prefer using MediaPipe without
@@ -57,7 +57,7 @@ Please verify all the necessary packages are installed.
 *   Android SDK Build-Tools 28 or 29
 *   Android SDK Platform-Tools 28 or 29
 *   Android SDK Tools 26.1.1
-*   Android NDK 17c or above
+*   Android NDK 19c or above
 
 ### Option 1: Build with Bazel in Command Line
 
@@ -111,7 +111,7 @@ app:
     *   Verify that Android SDK Build-Tools 28 or 29 is installed.
     *   Verify that Android SDK Platform-Tools 28 or 29 is installed.
     *   Verify that Android SDK Tools 26.1.1 is installed.
-    *   Verify that Android NDK 17c or above is installed.
+    *   Verify that Android NDK 19c or above is installed.
     *   Take note of the Android NDK Location, e.g.,
         `/usr/local/home/Android/Sdk/ndk-bundle` or
         `/usr/local/home/Android/Sdk/ndk/20.0.5594570`.

docs/getting_started/android_archive_library.md

@@ -37,7 +37,7 @@ each project.
     load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar")
 
     mediapipe_aar(
-        name = "mp_face_detection_aar",
+        name = "mediapipe_face_detection",
         calculators = ["//mediapipe/graphs/face_detection:mobile_calculators"],
     )
     ```
@@ -45,26 +45,29 @@ each project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
-        --fat_apk_cpu=arm64-v8a,armeabi-v7a --strip=ALWAYS \
-        //path/to/the/aar/build/file:aar_name
+    bazel build -c opt --strip=ALWAYS \
+        --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+        --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //path/to/the/aar/build/file:aar_name.aar
     ```
 
-    For the face detection AAR target we made in the step 1, run:
+    For the face detection AAR target we made in step 1, run:
 
     ```bash
-    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
-        //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
+    bazel build -c opt --strip=ALWAYS \
+        --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+        --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar
 
     # It should print:
-    # Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar up-to-date:
-    # bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    # Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar up-to-date:
+    # bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     ```
 
 3.  (Optional) Save the AAR to your preferred location.
 
     ```bash
-    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     /absolute/path/to/your/preferred/location
     ```
 
@@ -75,7 +78,7 @@ each project.
 2.  Copy the AAR into app/libs.
 
     ```bash
-    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mp_face_detection_aar.aar
+    cp bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example/mediapipe_face_detection.aar
     /path/to/your/app/libs/
     ```
 
@@ -92,29 +95,14 @@ each project.
     [the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite).
 
     ```bash
-    bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
-    cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
+    bazel build -c opt mediapipe/graphs/face_detection:face_detection_mobile_gpu_binary_graph
+    cp bazel-bin/mediapipe/graphs/face_detection/face_detection_mobile_gpu.binarypb /path/to/your/app/src/main/assets/
     cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/
     ```
 
     ![Screenshot](../images/mobile/assets_location.png)
 
-4.  Make app/src/main/jniLibs and copy OpenCV JNI libraries into
-    app/src/main/jniLibs.
-
-    MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
-    files into app/src/main/jniLibs. You can download the official OpenCV
-    Android SDK from
-    [here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
-    and run:
-
-    ```bash
-    cp -R ~/Downloads/OpenCV-android-sdk/sdk/native/libs/arm* /path/to/your/app/src/main/jniLibs/
-    ```
-
-    ![Screenshot](../images/mobile/android_studio_opencv_location.png)
-
-5.  Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
+4.  Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
 
     ```
     dependencies {
@@ -136,10 +124,14 @@ each project.
         implementation "androidx.camera:camera-core:$camerax_version"
         implementation "androidx.camera:camera-camera2:$camerax_version"
         implementation "androidx.camera:camera-lifecycle:$camerax_version"
+        // AutoValue
+        def auto_value_version = "1.6.4"
+        implementation "com.google.auto.value:auto-value-annotations:$auto_value_version"
+        annotationProcessor "com.google.auto.value:auto-value:$auto_value_version"
     }
     ```
 
-6.  Follow our Android app examples to use MediaPipe in Android Studio for your
+5.  Follow our Android app examples to use MediaPipe in Android Studio for your
     use case. If you are looking for an example, a face detection example can be
     found
     [here](https://github.com/jiuqiant/mediapipe_face_detection_aar_example) and

docs/getting_started/install.md

@@ -25,25 +25,11 @@ install --user six`.
 
 ## Installing on Debian and Ubuntu
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install Bazel 3.4 or higher.
-
-    For Nvidia Jetson and Raspberry Pi devices with aarch64 Linux, Bazel needs
-    to be built from source:
-
-    ```bash
-    # For Bazel 3.4.1
-    mkdir $HOME/bazel-3.4.1
-    cd $HOME/bazel-3.4.1
-    wget https://github.com/bazelbuild/bazel/releases/download/3.4.1/bazel-3.4.1-dist.zip
-    sudo apt-get install build-essential openjdk-8-jdk python zip unzip
-    unzip bazel-3.4.1-dist.zip
-    env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
-    sudo cp output/bazel /usr/local/bin/
-    ```
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -207,11 +193,11 @@ build issues.
 
 **Disclaimer**: Running MediaPipe on CentOS is experimental.
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel 3.4 or higher.
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -336,11 +322,11 @@ build issues.
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
         Tools by `xcode-select --install`.
 
-2.  Install Bazel.
+2.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install Bazel 3.4 or higher.
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 3.  Checkout MediaPipe repository.
 
@@ -353,7 +339,7 @@ build issues.
 4.  Install OpenCV and FFmpeg.
 
     Option 1. Use HomeBrew package manager tool to install the pre-compiled
-    OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
+    OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
 
     ```bash
     $ brew install opencv@3
@@ -484,29 +470,36 @@ next section.
 
 4.  Install Visual C++ Build Tools 2019 and WinSDK
 
-    Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
-    build tools, and install Microsoft Visual C++ 2019 Redistributable and
-    Microsoft Build Tools 2019.
+    Go to
+    [the VisualStudio website](https://visualstudio.microsoft.com/visual-cpp-build-tools),
+    download build tools, and install Microsoft Visual C++ 2019 Redistributable
+    and Microsoft Build Tools 2019.
 
     Download the WinSDK from
-    https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
-    install.
+    [the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
+    and install.
 
-5.  Install Bazel and add the location of the Bazel executable to the `%PATH%`
-    environment variable.
+5.  Install Bazel or Bazelisk and add the location of the Bazel executable to
+    the `%PATH%` environment variable.
 
-    Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.4 or higher.
+    Option 1. Follow
+    [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
+    to install Bazel 3.7.2 or higher.
 
-6.  Set Bazel variables.
+    Option 2. Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
+
+6.  Set Bazel variables. Learn more details about
+    ["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
+    in the Bazel official documentation.
 
     ```
-    # Find the exact paths and version numbers from your local version.
+    # Please find the exact paths and version numbers from your local version.
     C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
     C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
-    C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
-    C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
+    C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
+    C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
     ```
 
 7.  Checkout MediaPipe repository.
@@ -593,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel.
+5.  Install Bazelisk.
 
-    ```bash
-    username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/3.4.1/release/bazel-3.4.1-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/3.4.1 && \
-    chmod 755 bazel-3.4.1-installer-linux-x86_64.sh && \
-    sudo ./bazel-3.4.1-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.1 && \
-    source /usr/local/bazel/3.4.1/lib/bazel/bin/bazel-complete.bash
-
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.1/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/3.4.1/lib/bazel/bin/bazel'
-    ```
+    Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 6.  Checkout MediaPipe repository.
 
@@ -753,7 +738,7 @@ common build issues.
     root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk.sh
 
     # Should print:
-    # Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b
+    # Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r19c
     # Set android_ndk_repository and android_sdk_repository in WORKSPACE
     # Done
 

docs/getting_started/javascript.md

@@ -16,13 +16,14 @@ nav_order: 4
 
 MediaPipe currently offers the following solutions:
 
-Solution          | NPM Package                   | Example
------------------ | ----------------------------- | -------
-[Face Mesh][F-pg] | [@mediapipe/face_mesh][F-npm] | [mediapipe.dev/demo/face_mesh][F-demo]
-[Face Detection][Fd-pg] | [@mediapipe/face_detection][Fd-npm] | [mediapipe.dev/demo/face_detection][Fd-demo]
-[Hands][H-pg]     | [@mediapipe/hands][H-npm]     | [mediapipe.dev/demo/hands][H-demo]
-[Holistic][Ho-pg] | [@mediapipe/holistic][Ho-npm] | [mediapipe.dev/demo/holistic][Ho-demo]
-[Pose][P-pg]      | [@mediapipe/pose][P-npm]      | [mediapipe.dev/demo/pose][P-demo]
+Solution                    | NPM Package                             | Example
+--------------------------- | --------------------------------------- | -------
+[Face Mesh][F-pg]           | [@mediapipe/face_mesh][F-npm]           | [mediapipe.dev/demo/face_mesh][F-demo]
+[Face Detection][Fd-pg]     | [@mediapipe/face_detection][Fd-npm]     | [mediapipe.dev/demo/face_detection][Fd-demo]
+[Hands][H-pg]               | [@mediapipe/hands][H-npm]               | [mediapipe.dev/demo/hands][H-demo]
+[Holistic][Ho-pg]           | [@mediapipe/holistic][Ho-npm]           | [mediapipe.dev/demo/holistic][Ho-demo]
+[Pose][P-pg]                | [@mediapipe/pose][P-npm]                | [mediapipe.dev/demo/pose][P-demo]
+[Selfie Segmentation][S-pg] | [@mediapipe/selfie_segmentation][S-npm] | [mediapipe.dev/demo/selfie_segmentation][S-demo]
 
 Click on a solution link above for more information, including API and code
 snippets.
@@ -67,28 +68,33 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 [Fd-pg]: ../solutions/face_detection#javascript-solution-api
 [H-pg]: ../solutions/hands#javascript-solution-api
 [P-pg]: ../solutions/pose#javascript-solution-api
+[S-pg]: ../solutions/selfie_segmentation#javascript-solution-api
 [Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic
 [F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh
 [Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
 [H-npm]: https://www.npmjs.com/package/@mediapipe/hands
 [P-npm]: https://www.npmjs.com/package/@mediapipe/pose
-[draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
-[cam-npm]: https://www.npmjs.com/package/@mediapipe/pose
-[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
+[S-npm]: https://www.npmjs.com/package/@mediapipe/selfie_segmentation
+[draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
+[cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
+[ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
 [Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
 [F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
 [Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
 [H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands
 [P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose
+[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/selfie_segmentation
 [Ho-pen]: https://code.mediapipe.dev/codepen/holistic
 [F-pen]: https://code.mediapipe.dev/codepen/face_mesh
 [Fd-pen]: https://code.mediapipe.dev/codepen/face_detection
 [H-pen]: https://code.mediapipe.dev/codepen/hands
 [P-pen]: https://code.mediapipe.dev/codepen/pose
+[S-pen]: https://code.mediapipe.dev/codepen/selfie_segmentation
 [Ho-demo]: https://mediapipe.dev/demo/holistic
 [F-demo]: https://mediapipe.dev/demo/face_mesh
 [Fd-demo]: https://mediapipe.dev/demo/face_detection
 [H-demo]: https://mediapipe.dev/demo/hands
 [P-demo]: https://mediapipe.dev/demo/pose
+[S-demo]: https://mediapipe.dev/demo/selfie_segmentation
 [npm]: https://www.npmjs.com/package/@mediapipe
 [codepen]: https://code.mediapipe.dev/codepen

docs/getting_started/python.md

@@ -26,7 +26,7 @@ You can, for instance, activate a Python virtual environment:
 $ python3 -m venv mp_env && source mp_env/bin/activate
 ```
 
-Install MediaPipe Python package and start Python intepreter:
+Install MediaPipe Python package and start Python interpreter:
 
 ```bash
 (mp_env)$ pip install mediapipe
@@ -51,6 +51,7 @@ details in each solution via the links below:
 *   [MediaPipe Holistic](../solutions/holistic#python-solution-api)
 *   [MediaPipe Objectron](../solutions/objectron#python-solution-api)
 *   [MediaPipe Pose](../solutions/pose#python-solution-api)
+*   [MediaPipe Selfie Segmentation](../solutions/selfie_segmentation#python-solution-api)
 
 ## MediaPipe on Google Colab
 
@@ -62,6 +63,7 @@ details in each solution via the links below:
 *   [MediaPipe Pose Colab](https://mediapipe.page.link/pose_py_colab)
 *   [MediaPipe Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
 *   [MediaPipe Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
+*   [MediaPipe Selfie Segmentation Colab](https://mediapipe.page.link/selfie_segmentation_py_colab)
 
 ## MediaPipe Python Framework
 

docs/getting_started/troubleshooting.md

@@ -97,6 +97,49 @@ linux_opencv/macos_opencv/windows_opencv.BUILD files for your local opencv
 libraries. [This GitHub issue](https://github.com/google/mediapipe/issues/666)
 may also help.
 
+## Python pip install failure
+
+The error message:
+
+```
+ERROR: Could not find a version that satisfies the requirement mediapipe
+ERROR: No matching distribution found for mediapipe
+```
+
+after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system.
+Please note that MediaPipe Python PyPI officially supports the **64-bit**
+version of Python 3.7 and above on the following OS:
+
+-   x86_64 Linux
+-   x86_64 macOS 10.15+
+-   amd64 Windows
+
+If the OS is currently supported and you still see this error, please make sure
+that both the Python and pip binary are for Python 3.7 and above. Otherwise,
+please consider building the MediaPipe Python package locally by following the
+instructions [here](python.md#building-mediapipe-python-package).
+
+## Python DLL load failure on Windows
+
+The error message:
+
+```
+ImportError: DLL load failed: The specified module could not be found
+```
+
+usually indicates that the local Windows system is missing Visual C++
+redistributable packages and/or Visual C++ runtime DLLs. This can be solved by
+either installing the official
+[vc_redist.x64.exe](https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0)
+or installing the "msvc-runtime" Python package by running
+
+```bash
+$ python -m pip install msvc-runtime
+```
+
+Please note that the "msvc-runtime" Python package is not released or maintained
+by Microsoft.
+
 ## Native method not found
 
 The error message:

docs/index.md

@@ -40,11 +40,12 @@ Hair Segmentation
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Pose](https://google.github.io/mediapipe/solutions/pose)                                       | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Holistic](https://google.github.io/mediapipe/solutions/holistic)                               | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
+[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation)         | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)             | ✅                                                               |                                                         | ✅                                                       |                                                               |                                                               |
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/solutions/autoflip.md

@@ -2,7 +2,7 @@
 layout: default
 title: AutoFlip (Saliency-aware Video Cropping)
 parent: Solutions
-nav_order: 13
+nav_order: 14
 ---
 
 # AutoFlip: Saliency-aware Video Cropping

docs/solutions/box_tracking.md

@@ -2,7 +2,7 @@
 layout: default
 title: Box Tracking
 parent: Solutions
-nav_order: 9
+nav_order: 10
 ---
 
 # MediaPipe Box Tracking

docs/solutions/face_detection.md

@@ -68,7 +68,7 @@ normalized to `[0.0, 1.0]` by the image width and height respectively.
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
@@ -77,12 +77,14 @@ Supported configuration options:
 ```python
 import cv2
 import mediapipe as mp
-mp_face_detction = mp.solutions.face_detection
+mp_face_detection = mp.solutions.face_detection
+mp_drawing = mp.solutions.drawing_utils
 
 # For static images:
+IMAGE_FILES = []
 with mp_face_detection.FaceDetection(
     min_detection_confidence=0.5) as face_detection:
-  for idx, file in enumerate(file_list):
+  for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     # Convert the BGR image to RGB and process it with MediaPipe Face Detection.
     results = face_detection.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
@@ -183,8 +185,8 @@ function onResults(results) {
   canvasCtx.restore();
 }
 
-const faceDetection = new Objectron({locateFile: (file) => {
-  return `https://cdn.jsdelivr.net/npm/@mediapipe/objectr[email protected]/${file}`;
+const faceDetection = new FaceDetection({locateFile: (file) => {
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/face_detecti[email protected]/${file}`;
 }});
 faceDetection.setOptions({
   minDetectionConfidence: 0.5

docs/solutions/face_mesh.md

@@ -265,7 +265,7 @@ magnitude of `z` uses roughly the same scale as `x`.
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
@@ -281,12 +281,13 @@ mp_drawing = mp.solutions.drawing_utils
 mp_face_mesh = mp.solutions.face_mesh
 
 # For static images:
+IMAGE_FILES = []
 drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 with mp_face_mesh.FaceMesh(
     static_image_mode=True,
     max_num_faces=1,
     min_detection_confidence=0.5) as face_mesh:
-  for idx, file in enumerate(file_list):
+  for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     # Convert the BGR image to RGB before processing.
     results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))

docs/solutions/hair_segmentation.md

@@ -2,7 +2,7 @@
 layout: default
 title: Hair Segmentation
 parent: Solutions
-nav_order: 7
+nav_order: 8
 ---
 
 # MediaPipe Hair Segmentation

docs/solutions/hands.md

@@ -206,7 +206,7 @@ is not the case, please swap the handedness output in the application.
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
@@ -222,11 +222,12 @@ mp_drawing = mp.solutions.drawing_utils
 mp_hands = mp.solutions.hands
 
 # For static images:
+IMAGE_FILES = []
 with mp_hands.Hands(
     static_image_mode=True,
     max_num_hands=2,
     min_detection_confidence=0.5) as hands:
-  for idx, file in enumerate(file_list):
+  for idx, file in enumerate(IMAGE_FILES):
     # Read an image, flip it around y-axis for correct handedness output (see
     # above).
     image = cv2.flip(cv2.imread(file), 1)

docs/solutions/holistic.md

@@ -135,12 +135,11 @@ another detection until it loses track, on reducing computation and latency. If
 set to `true`, person detection runs every input image, ideal for processing a
 batch of static, possibly unrelated, images. Default to `false`.
 
-#### upper_body_only
+#### model_complexity
 
-If set to `true`, the solution outputs only the 25 upper-body pose landmarks
-(535 in total) instead of the full set of 33 pose landmarks (543 in total). Note
-that upper-body-only prediction may be more accurate for use cases where the
-lower-body parts are mostly out of view. Default to `false`.
+Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
+well as inference latency generally go up with the model complexity. Default to
+`1`.
 
 #### smooth_landmarks
 
@@ -202,12 +201,12 @@ A list of 21 hand landmarks on the right hand, in the same representation as
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
-*   [upper_body_only](#upper_body_only)
+*   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
@@ -219,8 +218,11 @@ mp_drawing = mp.solutions.drawing_utils
 mp_holistic = mp.solutions.holistic
 
 # For static images:
-with mp_holistic.Holistic(static_image_mode=True) as holistic:
-  for idx, file in enumerate(file_list):
+IMAGE_FILES = []
+with mp_holistic.Holistic(
+    static_image_mode=True,
+    model_complexity=2) as holistic:
+  for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
     # Convert the BGR image to RGB before processing.
@@ -240,8 +242,6 @@ with mp_holistic.Holistic(static_image_mode=True) as holistic:
         annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
     mp_drawing.draw_landmarks(
         annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-    # Use mp_holistic.UPPER_BODY_POSE_CONNECTIONS for drawing below when
-    # upper_body_only is set to True.
     mp_drawing.draw_landmarks(
         annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
@@ -291,7 +291,7 @@ and the following usage example.
 
 Supported configuration options:
 
-*   [upperBodyOnly](#upper_body_only)
+*   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
@@ -348,7 +348,7 @@ const holistic = new Holistic({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`;
 }});
 holistic.setOptions({
-  upperBodyOnly: false,
+  modelComplexity: 1,
   smoothLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5

docs/solutions/instant_motion_tracking.md

@@ -2,7 +2,7 @@
 layout: default
 title: Instant Motion Tracking
 parent: Solutions
-nav_order: 10
+nav_order: 11
 ---
 
 # MediaPipe Instant Motion Tracking

docs/solutions/knift.md

@@ -2,7 +2,7 @@
 layout: default
 title: KNIFT (Template-based Feature Matching)
 parent: Solutions
-nav_order: 12
+nav_order: 13
 ---
 
 # MediaPipe KNIFT

docs/solutions/media_sequence.md

@@ -2,7 +2,7 @@
 layout: default
 title: Dataset Preparation with MediaSequence
 parent: Solutions
-nav_order: 14
+nav_order: 15
 ---
 
 # Dataset Preparation with MediaSequence

docs/solutions/models.md

@@ -15,11 +15,16 @@ nav_order: 30
 ### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
 
 *   Face detection model for front-facing/selfie camera:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
-    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite),
+    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite),
+    [Model card](https://mediapipe.page.link/blazeface-mc)
 *   Face detection model for back-facing camera:
-    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
-*   [Model card](https://mediapipe.page.link/blazeface-mc)
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back.tflite),
+    [Model card](https://mediapipe.page.link/blazeface-back-mc)
+*   Face detection model for back-facing camera (sparse):
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_back_sparse.tflite),
+    [Model card](https://mediapipe.page.link/blazeface-back-sparse-mc)
+
 
 ### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
 
@@ -49,10 +54,10 @@ nav_order: 30
 
 *   Pose detection model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
-*   Full-body pose landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full_body.tflite)
-*   Upper-body pose landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
+*   Pose landmark model:
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite),
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full.tflite),
+    [TFLite model (heavy)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite)
 *   [Model card](https://mediapipe.page.link/blazepose-mc)
 
 ### [Holistic](https://google.github.io/mediapipe/solutions/holistic)
@@ -60,6 +65,12 @@ nav_order: 30
 *   Hand recrop model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/hand_recrop.tflite)
 
+### [Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation)
+
+*   [TFLite model (general)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite)
+*   [TFLite model (landscape)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite)
+*   [Model card](https://mediapipe.page.link/selfiesegmentation-mc)
+
 ### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
 
 *   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)

docs/solutions/object_detection.md

@@ -2,7 +2,7 @@
 layout: default
 title: Object Detection
 parent: Solutions
-nav_order: 8
+nav_order: 9
 ---
 
 # MediaPipe Object Detection

docs/solutions/objectron.md

@@ -2,7 +2,7 @@
 layout: default
 title: Objectron (3D Object Detection)
 parent: Solutions
-nav_order: 11
+nav_order: 12
 ---
 
 # MediaPipe Objectron
@@ -277,7 +277,7 @@ following:
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
@@ -297,11 +297,12 @@ mp_drawing = mp.solutions.drawing_utils
 mp_objectron = mp.solutions.objectron
 
 # For static images:
+IMAGE_FILES = []
 with mp_objectron.Objectron(static_image_mode=True,
                             max_num_objects=5,
                             min_detection_confidence=0.5,
                             model_name='Shoe') as objectron:
-  for idx, file in enumerate(file_list):
+  for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     # Convert the BGR image to RGB and process it with MediaPipe Objectron.
     results = objectron.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
@@ -358,15 +359,17 @@ cap.release()
 ## Example Apps
 
 Please first see general instructions for
-[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
-how to build MediaPipe examples.
+[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
+[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
 
 Note: To visualize a graph, copy the graph and paste it into
 [MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Two-stage Objectron
+### Mobile
+
+#### Two-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -404,7 +407,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Single-stage Objectron
+#### Single-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -428,7 +431,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Assets
+#### Assets
 
 Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
  format into a custom .uuu format. This can be done for user assets as follows:
@@ -449,9 +452,35 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
 > absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
 
-### Coordinate Systems
 
-#### Object Coordinate
+### Desktop
+
+To build the application, run:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
+```
+
+To run the application, replace `<input video path>` and `<output video path>`
+in the command below with your own paths, and `<landmark model path>` and
+`<allowed labels>` with the following:
+
+Category | `<landmark model path>`                                                     | `<allowed labels>`
+:------- | :-------------------------------------------------------------------------- | :-----------------
+Shoe     | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
+Chair    | mediapipe/modules/objectron/object_detection_3d_chair.tflite    | Chair
+Cup      | mediapipe/modules/objectron/object_detection_3d_cup.tflite      | Mug
+Camera   | mediapipe/modules/objectron/object_detection_3d_camera.tflite   | Camera
+
+```
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
+  --input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
+```
+
+## Coordinate Systems
+
+### Object Coordinate
 
 Each object has its object coordinate frame. We use the below object coordinate
 definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
@@ -459,7 +488,7 @@ origin is at the center of the 3D bounding box.
 
 ![box_coordinate.svg](../images/box_coordinate.svg)
 
-#### Camera Coordinate
+### Camera Coordinate
 
 A 3D object is parameterized by its `scale` and `rotation`, `translation` with
 regard to the camera coordinate frame. In this API we use the below camera
@@ -476,7 +505,7 @@ camera frame by applying `rotation` and `translation`:
 landmarks_3d = rotation * scale * unit_box + translation
 ```
 
-#### NDC Space
+### NDC Space
 
 In this API we use
 [NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
@@ -495,7 +524,7 @@ y_ndc = -fy * Y / Z + py
 z_ndc = 1 / Z
 ```
 
-#### Pixel Space
+### Pixel Space
 
 In this API we set upper-left coner of an image as the origin of pixel
 coordinate. One can convert from NDC to pixel space as follows:
@@ -532,10 +561,11 @@ py = -py_pixel * 2.0 / image_height + 1.0
     [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
 *   Google AI Blog:
     [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
+*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
 *   Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
     Shape Supervision](https://arxiv.org/abs/2003.03522)
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
-    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
+    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
 *   [Models and model cards](./models.md#objectron)
 *   [Python Colab](https://mediapipe.page.link/objectron_py_colab)

docs/solutions/pose.md

@@ -30,8 +30,7 @@ overlay of digital content and information on top of the physical world in
 augmented reality.
 
 MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
-33 3D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
-frames utilizing our
+33 3D landmarks on the whole body from RGB video frames utilizing our
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 research that also powers the
 [ML Kit Pose Detection API](https://developers.google.com/ml-kit/vision/pose-detection).
@@ -40,9 +39,9 @@ environments for inference, whereas our method achieves real-time performance on
 most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
 [python](#python-solution-api) and even on the [web](#javascript-solution-api).
 
-![pose_tracking_upper_body_example.gif](../images/mobile/pose_tracking_upper_body_example.gif) |
-:--------------------------------------------------------------------------------------------: |
-*Fig 1. Example of MediaPipe Pose for upper-body pose tracking.*                               |
+![pose_tracking_example.gif](../images/mobile/pose_tracking_example.gif) |
+:----------------------------------------------------------------------: |
+*Fig 1. Example of MediaPipe Pose for pose tracking.*                    |
 
 ## ML Pipeline
 
@@ -77,6 +76,36 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
+## Pose Estimation Quality
+
+To evaluate the quality of our [models](./models.md#pose) against other
+well-performing publicly available solutions, we use three different validation
+datasets, representing different verticals: Yoga, Dance and HIIT. Each image
+contains only a single person located 2-4 meters from the camera. To be
+consistent with other solutions, we perform evaluation only for 17 keypoints
+from [COCO topology](https://cocodataset.org/#keypoints-2020).
+
+Method                                                                                                | Yoga <br/> [`mAP`] | Yoga <br/> [`[email protected]`] | Dance <br/> [`mAP`] | Dance <br/> [`[email protected]`] | HIIT <br/> [`mAP`] | HIIT <br/> [`[email protected]`]
+----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------:
+BlazePose.Heavy                                                                                       | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
+BlazePose.Full                                                                                        | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
+BlazePose.Lite                                                                                        | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
+[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
+[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
+
+![pose_tracking_pck_chart.png](../images/mobile/pose_tracking_pck_chart.png) |
+:--------------------------------------------------------------------------: |
+*Fig 2. Quality evaluation in [`[email protected]`].*                                  |
+
+We designed our models specifically for live perception use cases, so all of
+them work in real-time on the majority of modern devices.
+
+Method          | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
+--------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
+BlazePose.Heavy | 53 ms                                                                                        | 38 ms
+BlazePose.Full  | 25 ms                                                                                        | 27 ms
+BlazePose.Lite  | 20 ms                                                                                        | 25 ms
+
 ## Models
 
 ### Person/pose Detection Model (BlazePose Detector)
@@ -93,15 +122,12 @@ hip midpoints.
 
 ![pose_tracking_detector_vitruvian_man.png](../images/mobile/pose_tracking_detector_vitruvian_man.png) |
 :----------------------------------------------------------------------------------------------------: |
-*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
+*Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
 
 ### Pose Landmark Model (BlazePose GHUM 3D)
 
-The landmark model in MediaPipe Pose comes in two versions: a full-body model
-that predicts the location of 33 pose landmarks (see figure below), and an
-upper-body version that only predicts the first 25. The latter may be more
-accurate than the former in scenarios where the lower-body parts are mostly out
-of view.
+The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
+(see figure below).
 
 Please find more detail in the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
@@ -111,7 +137,7 @@ this [paper](https://arxiv.org/abs/2006.10204) and
 
 ![pose_tracking_full_body_landmarks.png](../images/mobile/pose_tracking_full_body_landmarks.png) |
 :----------------------------------------------------------------------------------------------: |
-*Fig 3. 33 pose landmarks.*                                                                      |
+*Fig 4. 33 pose landmarks.*                                                                      |
 
 ## Solution APIs
 
@@ -129,12 +155,11 @@ until it loses track, on reducing computation and latency. If set to `true`,
 person detection runs every input image, ideal for processing a batch of static,
 possibly unrelated, images. Default to `false`.
 
-#### upper_body_only
+#### model_complexity
 
-If set to `true`, the solution outputs only the 25 upper-body pose landmarks.
-Otherwise, it outputs the full set of 33 pose landmarks. Note that
-upper-body-only prediction may be more accurate for use cases where the
-lower-body parts are mostly out of view. Default to `false`.
+Complexity of the pose landmark model: `0`, `1` or `2`. Landmark accuracy as
+well as inference latency generally go up with the model complexity. Default to
+`1`.
 
 #### smooth_landmarks
 
@@ -162,7 +187,7 @@ Naming style may differ slightly across platforms/languages.
 
 #### pose_landmarks
 
-A list of pose landmarks. Each lanmark consists of the following:
+A list of pose landmarks. Each landmark consists of the following:
 
 *   `x` and `y`: Landmark coordinates normalized to `[0.0, 1.0]` by the image
     width and height respectively.
@@ -170,9 +195,6 @@ A list of pose landmarks. Each lanmark consists of the following:
     being the origin, and the smaller the value the closer the landmark is to
     the camera. The magnitude of `z` uses roughly the same scale as `x`.
 
-    Note: `z` is predicted only in full-body mode, and should be discarded when
-    [upper_body_only](#upper_body_only) is `true`.
-
 *   `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the
     landmark being visible (present and not occluded) in the image.
 
@@ -180,12 +202,12 @@ A list of pose landmarks. Each lanmark consists of the following:
 
 Please first follow general [instructions](../getting_started/python.md) to
 install MediaPipe Python package, then learn more in the companion
-[Python Colab](#resources) and the following usage example.
+[Python Colab](#resources) and the usage example below.
 
 Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
-*   [upper_body_only](#upper_body_only)
+*   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
@@ -197,9 +219,12 @@ mp_drawing = mp.solutions.drawing_utils
 mp_pose = mp.solutions.pose
 
 # For static images:
+IMAGE_FILES = []
 with mp_pose.Pose(
-    static_image_mode=True, min_detection_confidence=0.5) as pose:
-  for idx, file in enumerate(file_list):
+    static_image_mode=True,
+    model_complexity=2,
+    min_detection_confidence=0.5) as pose:
+  for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
     # Convert the BGR image to RGB before processing.
@@ -214,8 +239,6 @@ with mp_pose.Pose(
     )
     # Draw pose landmarks on the image.
     annotated_image = image.copy()
-    # Use mp_pose.UPPER_BODY_POSE_CONNECTIONS for drawing below when
-    # upper_body_only is set to True.
     mp_drawing.draw_landmarks(
         annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
@@ -259,7 +282,7 @@ and the following usage example.
 
 Supported configuration options:
 
-*   [upperBodyOnly](#upper_body_only)
+*   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
@@ -306,7 +329,7 @@ const pose = new Pose({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/pose/${file}`;
 }});
 pose.setOptions({
-  upperBodyOnly: false,
+  modelComplexity: 1,
   smoothLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
@@ -347,16 +370,6 @@ to visualize its associated subgraphs, please see
 *   iOS target:
     [`mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp`](http:/mediapipe/examples/ios/posetrackinggpu/BUILD)
 
-#### Upper-body Only
-
-*   Graph:
-    [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
-*   Android target:
-    [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1uKc6T7KSuA0Mlq2URi5YookHu0U3yoh_/view?usp=sharing)
-    [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu:upperbodyposetrackinggpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/upperbodyposetrackinggpu/BUILD)
-*   iOS target:
-    [`mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp`](http:/mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD)
-
 ### Desktop
 
 Please first see general instructions for [desktop](../getting_started/cpp.md)
@@ -375,19 +388,6 @@ on how to build MediaPipe examples.
     *   Target:
         [`mediapipe/examples/desktop/pose_tracking:pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/pose_tracking/BUILD)
 
-#### Upper-body Only
-
-*   Running on CPU
-    *   Graph:
-        [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_cpu.pbtxt)
-    *   Target:
-        [`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
-*   Running on GPU
-    *   Graph:
-        [`mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/pose_tracking/upper_body_pose_tracking_gpu.pbtxt)
-    *   Target:
-        [`mediapipe/examples/desktop/upper_body_pose_tracking:upper_body_pose_tracking_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/upper_body_pose_tracking/BUILD)
-
 ## Resources
 
 *   Google AI Blog:
@@ -398,3 +398,6 @@ on how to build MediaPipe examples.
 *   [Models and model cards](./models.md#pose)
 *   [Web demo](https://code.mediapipe.dev/codepen/pose)
 *   [Python Colab](https://mediapipe.page.link/pose_py_colab)
+
+[`mAP`]: https://cocodataset.org/#keypoints-eval
+[`[email protected]`]: https://github.com/cbsudux/Human-Pose-Estimation-101

docs/solutions/pose_classification.md

@@ -25,10 +25,11 @@ One of the applications
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 can enable is fitness. More specifically - pose classification and repetition
 counting. In this section we'll provide basic guidance on building a custom pose
-classifier with the help of [Colabs](#colabs) and wrap it in a simple
-[fitness app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)
-powered by [ML Kit](https://developers.google.com/ml-kit). Push-ups and squats
-are used for demonstration purposes as the most common exercises.
+classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
+demo within
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
+Push-ups and squats are used for demonstration purposes as the most common
+exercises.
 
 ![pose_classification_pushups_and_squats.gif](../images/mobile/pose_classification_pushups_and_squats.gif) |
 :--------------------------------------------------------------------------------------------------------: |
@@ -47,7 +48,7 @@ determines the object's class based on the closest samples in the training set.
     classifier and form a training set using these [Colabs](#colabs),
 3.  Perform the classification itself followed by repetition counting (e.g., in
     the
-    [ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app)).
+    [ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
 
 ## Training Set
 
@@ -76,7 +77,7 @@ video right in the Colab.
 
 Code of the classifier is available both in the
 [`Pose Classification Colab (Extended)`] and in the
-[ML Kit demo app](https://mediapipe.page.link/mlkit-pose-classification-demo-app).
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
 Please refer to them for details of the approach described below.
 
 The k-NN algorithm used for pose classification requires a feature vector
@@ -127,11 +128,13 @@ where the pose class and the counter can't be changed.
 
 ## Future Work
 
-We are actively working on improving BlazePose GHUM 3D's Z prediction. It will
-allow us to use joint angles in the feature vectors, which are more natural and
-easier to configure (although distances can still be useful to detect touches
-between body parts) and to perform rotation normalization of poses and reduce
-the number of camera angles required for accurate k-NN classification.
+We are actively working on improving
+[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
+prediction. It will allow us to use joint angles in the feature vectors, which
+are more natural and easier to configure (although distances can still be useful
+to detect touches between body parts) and to perform rotation normalization of
+poses and reduce the number of camera angles required for accurate k-NN
+classification.
 
 ## Colabs
 

docs/solutions/selfie_segmentation.md

@@ -0,0 +1,289 @@
+---
+layout: default
+title: Selfie Segmentation
+parent: Solutions
+nav_order: 7
+---
+
+# MediaPipe Selfie Segmentation
+{: .no_toc }
+
+<details close markdown="block">
+  <summary>
+    Table of contents
+  </summary>
+  {: .text-delta }
+1. TOC
+{:toc}
+</details>
+---
+
+## Overview
+
+*Fig 1. Example of MediaPipe Selfie Segmentation.* |
+:------------------------------------------------: |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/selfie_segmentation_web.mp4" type="video/mp4"></video> |
+
+MediaPipe Selfie Segmentation segments the prominent humans in the scene. It can
+run in real-time on both smartphones and laptops. The intended use cases include
+selfie effects and video conferencing, where the person is close (< 2m) to the
+camera.
+
+## Models
+
+In this solution, we provide two models: general and landscape. Both models are
+based on
+[MobileNetV3](https://ai.googleblog.com/2019/11/introducing-next-generation-on-device.html),
+with modifications to make them more efficient. The general model operates on a
+256x256x3 (HWC) tensor, and outputs a 256x256x1 tensor representing the
+segmentation mask. The landscape model is similar to the general model, but
+operates on a 144x256x3 (HWC) tensor. It has fewer FLOPs than the general model,
+and therefore, runs faster. Note that MediaPipe Selfie Segmentation
+automatically resizes the input image to the desired tensor dimension before
+feeding it into the ML models.
+
+The general model is also powering
+[ML Kit](https://developers.google.com/ml-kit/vision/selfie-segmentation), and a
+variant of the landscape model is powering
+[Google Meet](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html).
+Please find more detail about the models in the
+[model card](./models.md#selfie-segmentation).
+
+## ML Pipeline
+
+The pipeline is implemented as a MediaPipe
+[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt)
+that uses a
+[selfie segmentation subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation_gpu.pbtxt)
+from the
+[selfie segmentation module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation).
+
+Note: To visualize a graph, copy the graph and paste it into
+[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
+to visualize its associated subgraphs, please see
+[visualizer documentation](../tools/visualizer.md).
+
+## Solution APIs
+
+### Cross-platform Configuration Options
+
+Naming style and availability may differ slightly across platforms/languages.
+
+#### model_selection
+
+An integer index `0` or `1`. Use `0` to select the general model, and `1` to
+select the landscape model (see details in [Models](#models)). Default to `0` if
+not specified.
+
+### Output
+
+Naming style may differ slightly across platforms/languages.
+
+#### segmentation_mask
+
+The output segmentation mask, which has the same dimension as the input image.
+
+### Python Solution API
+
+Please first follow general [instructions](../getting_started/python.md) to
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the usage example below.
+
+Supported configuration options:
+
+*   [model_selection](#model_selection)
+
+```python
+import cv2
+import mediapipe as mp
+mp_drawing = mp.solutions.drawing_utils
+mp_selfie_segmentation = mp.solutions.selfie_segmentation
+
+# For static images:
+IMAGE_FILES = []
+BG_COLOR = (192, 192, 192) # gray
+MASK_COLOR = (255, 255, 255) # white
+with mp_selfie_segmentation.SelfieSegmentation(
+    model_selection=0) as selfie_segmentation:
+  for idx, file in enumerate(IMAGE_FILES):
+    image = cv2.imread(file)
+    image_height, image_width, _ = image.shape
+    # Convert the BGR image to RGB before processing.
+    results = selfie_segmentation.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+
+    # Draw selfie segmentation on the background image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
+    # Generate solid color images for showing the output selfie segmentation mask.
+    fg_image = np.zeros(image.shape, dtype=np.uint8)
+    fg_image[:] = MASK_COLOR
+    bg_image = np.zeros(image.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    output_image = np.where(condition, fg_image, bg_image)
+    cv2.imwrite('/tmp/selfie_segmentation_output' + str(idx) + '.png', output_image)
+
+# For webcam input:
+BG_COLOR = (192, 192, 192) # gray
+cap = cv2.VideoCapture(0)
+with mp_selfie_segmentation.SelfieSegmentation(
+    model_selection=1) as selfie_segmentation:
+  bg_image = None
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
+
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = selfie_segmentation.process(image)
+
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+
+    # Draw selfie segmentation on the background image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack(
+      (results.segmentation_mask,) * 3, axis=-1) > 0.1
+    # The background can be customized.
+    #   a) Load an image (with the same width and height of the input image) to
+    #      be the background, e.g., bg_image = cv2.imread('/path/to/image/file')
+    #   b) Blur the input image by applying image filtering, e.g.,
+    #      bg_image = cv2.GaussianBlur(image,(55,55),0)
+    if bg_image is None:
+      bg_image = np.zeros(image.shape, dtype=np.uint8)
+      bg_image[:] = BG_COLOR
+    output_image = np.where(condition, image, bg_image)
+
+    cv2.imshow('MediaPipe Selfie Segmentation', output_image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
+cap.release()
+```
+
+### JavaScript Solution API
+
+Please first see general [introduction](../getting_started/javascript.md) on
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
+
+Supported configuration options:
+
+*   [modelSelection](#model_selection)
+
+```html
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/selfie_segmentation.js" crossorigin="anonymous"></script>
+</head>
+
+<body>
+  <div class="container">
+    <video class="input_video"></video>
+    <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+  </div>
+</body>
+</html>
+```
+
+```javascript
+<script type="module">
+const videoElement = document.getElementsByClassName('input_video')[0];
+const canvasElement = document.getElementsByClassName('output_canvas')[0];
+const canvasCtx = canvasElement.getContext('2d');
+
+function onResults(results) {
+  canvasCtx.save();
+  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(results.segmentationMask, 0, 0,
+                      canvasElement.width, canvasElement.height);
+
+  // Only overwrite existing pixels.
+  canvasCtx.globalCompositeOperation = 'source-in';
+  canvasCtx.fillStyle = '#00FF00';
+  canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
+
+  // Only overwrite missing pixels.
+  canvasCtx.globalCompositeOperation = 'destination-atop';
+  canvasCtx.drawImage(
+      results.image, 0, 0, canvasElement.width, canvasElement.height);
+
+  canvasCtx.restore();
+}
+
+const selfieSegmentation = new SelfieSegmentation({locateFile: (file) => {
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`;
+}});
+selfieSegmentation.setOptions({
+  modelSelection: 1,
+});
+selfieSegmentation.onResults(onResults);
+
+const camera = new Camera(videoElement, {
+  onFrame: async () => {
+    await selfieSegmentation.send({image: videoElement});
+  },
+  width: 1280,
+  height: 720
+});
+camera.start();
+</script>
+```
+
+## Example Apps
+
+Please first see general instructions for
+[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
+[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
+
+Note: To visualize a graph, copy the graph and paste it into
+[MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
+to visualize its associated subgraphs, please see
+[visualizer documentation](../tools/visualizer.md).
+
+### Mobile
+
+*   Graph:
+    [`mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt)
+*   Android target:
+    [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1DoeyGzMmWUsjfVgZfGGecrn7GKzYcEAo/view?usp=sharing)
+    [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu:selfiesegmentationgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD)
+*   iOS target:
+    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](http:/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
+
+### Desktop
+
+Please first see general instructions for [desktop](../getting_started/cpp.md)
+on how to build MediaPipe examples.
+
+*   Running on CPU
+    *   Graph:
+        [`mediapipe/graphs/selfie_segmentation/selfie_segmentation_cpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/selfie_segmentation/selfie_segmentation_cpu.pbtxt)
+    *   Target:
+        [`mediapipe/examples/desktop/selfie_segmentation:selfie_segmentation_cpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/selfie_segmentation/BUILD)
+*   Running on GPU
+    *   Graph:
+        [`mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt)
+    *   Target:
+        [`mediapipe/examples/desktop/selfie_segmentation:selfie_segmentation_gpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/selfie_segmentation/BUILD)
+
+## Resources
+
+*   Google AI Blog:
+    [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
+*   [ML Kit Selfie Segmentation API](https://developers.google.com/ml-kit/vision/selfie-segmentation)
+*   [Models and model cards](./models.md#selfie_segmentation)
+*   [Web demo](https://code.mediapipe.dev/codepen/selfie_segmentation)
+*   [Python Colab](https://mediapipe.page.link/selfie_segmentation_py_colab)

docs/solutions/solutions.md

@@ -24,11 +24,12 @@ has_toc: false
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Pose](https://google.github.io/mediapipe/solutions/pose)                                       | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Holistic](https://google.github.io/mediapipe/solutions/holistic)                               | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
+[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation)         | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)             | ✅                                                               |                                                         | ✅                                                       |                                                               |                                                               |
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/solutions/youtube_8m.md

@@ -2,7 +2,7 @@
 layout: default
 title: YouTube-8M Feature Extraction and Model Inference
 parent: Solutions
-nav_order: 15
+nav_order: 16
 ---
 
 # YouTube-8M Feature Extraction and Model Inference

docs/tools/tracing_and_profiling.md

@@ -41,6 +41,7 @@ profiler_config {
   trace_enabled: true
   enable_profiler: true
   trace_log_interval_count: 200
+  trace_log_path: "/sdcard/Download/"
 }
 ```
 
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
 
 *   On the desktop, this will be the `/tmp` directory.
 
-*   On Android, this will be the `/sdcard` directory.
+*   On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
 
 *   On iOS, this can be reached through XCode. Select "Window/Devices and
     Simulators" and select the "Devices" tab.
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     *   Include the line below in your `AndroidManifest.xml` file.
 
         ```xml
-        <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+        <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
         ```
 
     *   Grant the permission either upon first app launch, or by going into
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
     events to a trace log files at:
 
     ```bash
-    /sdcard/mediapipe_trace_0.binarypb
-    /sdcard/mediapipe_trace_1.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_0.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_1.binarypb
     ```
 
     After every 5 sec, writing shifts to a successive trace log file, such that
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
     trace files have been written to the device using adb shell.
 
     ```bash
-    adb shell "ls -la /sdcard/"
+    adb shell "ls -la /storage/emulated/0/Download"
     ```
 
-    On android, MediaPipe selects the external storage directory `/sdcard` for
+    On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
     trace logs. This directory can be overridden using the setting
     `trace_log_path`, like:
 
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     profiler_config {
       trace_enabled: true
       enable_profiler: true
-      trace_log_path: "/sdcard/profiles/"
+      trace_log_path: "/sdcard/Download/profiles/"
     }
     ```
 
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
 
     ```bash
     # from your terminal
-    adb pull /sdcard/mediapipe_trace_0.binarypb
+    adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
     # if successful you should see something like
     # /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
     ```

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -16,7 +16,7 @@
     "mediapipe/examples/ios/objectdetectiongpu/BUILD",
     "mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD",
     "mediapipe/examples/ios/posetrackinggpu/BUILD",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD",
+    "mediapipe/examples/ios/selfiesegmentationgpu/BUILD",
     "mediapipe/framework/BUILD",
     "mediapipe/gpu/BUILD",
     "mediapipe/objc/BUILD",
@@ -36,7 +36,7 @@
     "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
     "//mediapipe/examples/ios/objectdetectiontrackinggpu:ObjectDetectionTrackingGpuApp",
     "//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp",
-    "//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
+    "//mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp",
     "//mediapipe/objc:mediapipe_framework_ios"
   ],
   "optionSet" : {
@@ -105,7 +105,7 @@
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectiongpu",
     "mediapipe/examples/ios/posetrackinggpu",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu",
+    "mediapipe/examples/ios/selfiesegmentationgpu",
     "mediapipe/framework",
     "mediapipe/framework/deps",
     "mediapipe/framework/formats",
@@ -123,6 +123,7 @@
     "mediapipe/graphs/hand_tracking",
     "mediapipe/graphs/object_detection",
     "mediapipe/graphs/pose_tracking",
+    "mediapipe/graphs/selfie_segmentation",
     "mediapipe/models",
     "mediapipe/modules",
     "mediapipe/objc",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -22,7 +22,7 @@
     "mediapipe/examples/ios/objectdetectiongpu",
     "mediapipe/examples/ios/objectdetectiontrackinggpu",
     "mediapipe/examples/ios/posetrackinggpu",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu",
+    "mediapipe/examples/ios/selfiesegmentationgpu",
     "mediapipe/objc"
   ],
   "projectName" : "Mediapipe",

mediapipe/calculators/audio/BUILD

@@ -128,7 +128,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -147,7 +147,7 @@ cc_library(
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp/mfcc",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -168,7 +168,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:resampler",
         "@com_google_audio_tools//audio/dsp:resampler_q",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -208,7 +208,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@com_google_audio_tools//audio/dsp/spectrogram",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -228,7 +228,7 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -242,9 +242,9 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -261,7 +261,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -276,7 +276,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -296,7 +296,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:number_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -314,7 +314,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -333,7 +333,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -352,6 +352,6 @@ cc_test(
         "//mediapipe/framework/tool:validate_type",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:signal_vector_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
@@ -25,7 +25,7 @@ namespace mediapipe {
 
 TEST(AudioDecoderCalculatorTest, TestWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -34,7 +34,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -56,7 +56,7 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
 
 TEST(AudioDecoderCalculatorTest, Test48KWAV) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -65,7 +65,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -87,7 +87,7 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
 
 TEST(AudioDecoderCalculatorTest, TestMP3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -96,7 +96,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",
@@ -118,7 +118,7 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
 
 TEST(AudioDecoderCalculatorTest, TestAAC) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "AudioDecoderCalculator"
         input_side_packet: "INPUT_FILE_PATH:input_file_path"
         output_stream: "AUDIO:audio"
@@ -127,7 +127,7 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
           [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
             audio_stream { stream_index: 0 }
           }
-        })");
+        })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
       file::JoinPath("./",

mediapipe/calculators/core/BUILD

@@ -233,6 +233,22 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "concatenate_vector_calculator_hdr",
+    hdrs = ["concatenate_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "concatenate_vector_calculator",
     srcs = ["concatenate_vector_calculator.cc"],
@@ -414,7 +430,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -430,7 +446,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -450,6 +466,35 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "non_zero_calculator",
+    srcs = ["non_zero_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "non_zero_calculator_test",
+    size = "small",
+    srcs = ["non_zero_calculator_test.cc"],
+    deps = [
+        ":non_zero_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_test(
     name = "mux_calculator_test",
     srcs = ["mux_calculator_test.cc"],
@@ -651,6 +696,18 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "default_side_packet_calculator",
+    srcs = ["default_side_packet_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "side_packet_to_stream_calculator",
     srcs = ["side_packet_to_stream_calculator.cc"],
@@ -776,7 +833,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -793,7 +850,7 @@ cc_test(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -1024,7 +1081,7 @@ cc_library(
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/memory",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -70,7 +70,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           node {
@@ -90,7 +90,7 @@ class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:ints_plus_one"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));
@@ -197,7 +197,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "force_ints_to_be_timestamp_bound_update"
@@ -229,7 +229,7 @@ class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest
             input_stream: "ints_plus_one"
             output_stream: "ints_plus_one_passed_through"
           }
-        )");
+        )pb");
     tool::AddVectorSink("ints_plus_one_passed_through", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
@@ -338,7 +338,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
  protected:
   void SetUp() override {
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
-        R"(
+        R"pb(
           num_threads: 4
           input_stream: "ints"
           input_stream: "multiplier"
@@ -362,7 +362,7 @@ class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
             input_stream: "BATCH_END:timestamp"
             output_stream: "ITERABLE:multiplied_ints"
           }
-        )");
+        )pb");
     tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config));
     MP_ASSERT_OK(graph_.StartRun({}));

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -38,14 +38,14 @@ void AddInputVector(const std::vector<int>& input, int64 timestamp,
 
 TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {};
@@ -60,14 +60,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   std::vector<int> input = {0, 1, 2, 3};
@@ -85,14 +85,14 @@ TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         output_stream: "output_vector"
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
 
   {
@@ -133,7 +133,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "input_vector"
         node {
           calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
@@ -143,7 +143,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
             [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("output_vector", &graph_config, &outputs);
@@ -178,7 +178,7 @@ TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
 
 TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "TestClipIntVectorSizeCalculator"
         input_stream: "input_vector"
         input_side_packet: "max_vec_size"
@@ -186,7 +186,7 @@ TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) {
         options {
           [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   // This should override the default of 1 set in the options.
   runner.MutableSidePackets()->Index(0) = Adopt(new int(2));

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -392,7 +392,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         input_stream: "in_3"
@@ -403,7 +403,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
           input_stream: "in_3"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -456,7 +456,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -465,7 +465,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
           input_stream: "in_2"
           output_stream: "out"
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);
@@ -505,7 +505,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
    * The test needs to send packets that own the data.
    */
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: "in_1"
         input_stream: "in_2"
         node {
@@ -519,7 +519,7 @@ TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
             }
           }
         }
-      )");
+      )pb");
 
   std::vector<Packet> outputs;
   tool::AddVectorSink("out", &graph_config, &outputs);

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -62,7 +62,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -82,7 +82,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -111,7 +111,7 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
 
 TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -131,7 +131,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   MP_ASSERT_OK(graph.Initialize(graph_config));
   MP_ASSERT_OK(graph.StartRun({}));
@@ -152,7 +152,7 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:int_packet"
@@ -163,14 +163,14 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
   CalculatorGraphConfig graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         node {
           calculator: "ConstantSidePacketCalculator"
           output_side_packet: "PACKET:0:int_packet"
@@ -181,7 +181,7 @@ TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) {
             }
           }
         }
-      )");
+      )pb");
   CalculatorGraph graph;
   EXPECT_FALSE(graph.Initialize(graph_config).ok());
 }

mediapipe/calculators/core/default_side_packet_calculator.cc

@@ -0,0 +1,104 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+namespace {
+
+constexpr char kOptionalValueTag[] = "OPTIONAL_VALUE";
+constexpr char kDefaultValueTag[] = "DEFAULT_VALUE";
+constexpr char kValueTag[] = "VALUE";
+
+}  // namespace
+
+// Outputs side packet default value if optional value is not provided.
+//
+// This calculator utilizes the fact that MediaPipe automatically removes
+// optional side packets of the calculator configuration (i.e. OPTIONAL_VALUE).
+// And if it happens - returns default value, otherwise - returns optional
+// value.
+//
+// Input:
+//   OPTIONAL_VALUE (optional) - AnyType (but same type as DEFAULT_VALUE)
+//     Optional side packet value that is outputted by the calculator as is if
+//     provided.
+//
+//   DEFAULT_VALUE - AnyType
+//     Default side pack value that is outputted by the calculator if
+//     OPTIONAL_VALUE is not provided.
+//
+// Output:
+//   VALUE - AnyType (but same type as DEFAULT_VALUE)
+//     Either OPTIONAL_VALUE (if provided) or DEFAULT_VALUE (otherwise).
+//
+// Usage example:
+//   node {
+//     calculator: "DefaultSidePacketCalculator"
+//     input_side_packet: "OPTIONAL_VALUE:segmentation_mask_enabled_optional"
+//     input_side_packet: "DEFAULT_VALUE:segmentation_mask_enabled_default"
+//     output_side_packet: "VALUE:segmentation_mask_enabled"
+//   }
+class DefaultSidePacketCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(DefaultSidePacketCalculator);
+
+absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) {
+  RET_CHECK(cc->InputSidePackets().HasTag(kDefaultValueTag))
+      << "Default value must be provided";
+  cc->InputSidePackets().Tag(kDefaultValueTag).SetAny();
+
+  // Optional input side packet can be unspecified. In this case MediaPipe will
+  // remove it from the calculator config.
+  if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
+    cc->InputSidePackets()
+        .Tag(kOptionalValueTag)
+        .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag))
+        .Optional();
+  }
+
+  RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag));
+  cc->OutputSidePackets().Tag(kValueTag).SetSameAs(
+      &cc->InputSidePackets().Tag(kDefaultValueTag));
+
+  return absl::OkStatus();
+}
+
+absl::Status DefaultSidePacketCalculator::Open(CalculatorContext* cc) {
+  // If optional value is provided it is returned as the calculator output.
+  if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
+    auto& packet = cc->InputSidePackets().Tag(kOptionalValueTag);
+    cc->OutputSidePackets().Tag(kValueTag).Set(packet);
+    return absl::OkStatus();
+  }
+
+  // If no optional value
+  auto& packet = cc->InputSidePackets().Tag(kDefaultValueTag);
+  cc->OutputSidePackets().Tag(kValueTag).Set(packet);
+
+  return absl::OkStatus();
+}
+
+absl::Status DefaultSidePacketCalculator::Process(CalculatorContext* cc) {
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             max_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -51,7 +51,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -61,7 +61,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -76,7 +76,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -86,7 +86,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(
@@ -101,7 +101,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "DequantizeByteArrayCalculator"
         input_stream: "ENCODED:encoded"
         output_stream: "FLOAT_VECTOR:float_vector"
@@ -111,7 +111,7 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
             min_quantized_value: -2
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
   runner.MutableInputs()->Tag("ENCODED").packets.push_back(

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -57,7 +57,7 @@ namespace mediapipe {
 //
 // The "ALLOW" stream indicates the transition between accepting frames and
 // dropping frames.  "ALLOW = true" indicates the start of accepting frames
-// including the current timestamp, and "ALLOW = true" indicates the start of
+// including the current timestamp, and "ALLOW = false" indicates the start of
 // dropping frames including the current timestamp.
 //
 // FlowLimiterCalculator provides limited support for multiple input streams.

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -126,7 +126,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The LambdaCalculator is used to keep certain frames in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -143,7 +143,7 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
         input_stream: 'in_1_sampled'
         output_stream: 'out_1'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -271,7 +271,7 @@ REGISTER_CALCULATOR(DropCalculator);
 class FlowLimiterCalculatorTest : public testing::Test {
  protected:
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       node {
         calculator: 'FlowLimiterCalculator'
@@ -296,7 +296,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
         input_stream: 'PACKET:out_1_sampled'
         output_stream: 'PACKET:out_1'
       }
-    )");
+    )pb");
   }
 
   // Parse an absl::Time from RFC3339 format.
@@ -348,10 +348,10 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -419,11 +419,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -483,11 +483,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config = InflightGraphConfig();
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -548,7 +548,7 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -576,13 +576,13 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 1
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -657,7 +657,7 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
   SetUpInputData();
   SetUpSimulationClock();
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_1'
         input_stream: 'in_2'
         node {
@@ -685,13 +685,13 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
           input_stream: 'PACKET:out_1_sampled'
           output_stream: 'PACKET:out_1'
         }
-      )");
+      )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
     max_in_flight: 1
     max_in_queue: 0
     in_flight_timeout: 100000  # 100 ms
-  )");
+  )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -64,13 +64,13 @@ const char kMatrixText2[] =
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_side_packet: "MINUEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(
@@ -81,12 +81,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig) {
 
 TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_side_packet: "SUBTRAHEND:side_matrix"
         input_stream: "SUBTRAHEND:side_matrix2"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
   EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
@@ -95,12 +95,12 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "MINUEND:input_matrix"
         input_side_packet: "SUBTRAHEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);
@@ -124,12 +124,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "MatrixSubtractCalculator"
         input_stream: "SUBTRAHEND:input_matrix"
         input_side_packet: "MINUEND:side_matrix"
         output_stream: "output_matrix"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);

mediapipe/calculators/core/merge_calculator_test.cc

@@ -26,31 +26,33 @@ namespace {
 
 // Checks that the calculator fails if no input streams are provided.
 TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     output_stream: "merged_output"
-  )"));
+  )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner.Run().ok());
 }
 
 // Checks that the calculator fails with an incorrect number of output streams.
 TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
-  CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
-    calculator: "MergeCalculator"
-    input_stream: "input1"
-    input_stream: "input2"
-  )"));
+  CalculatorRunner runner1(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
+        calculator: "MergeCalculator"
+        input_stream: "input1"
+        input_stream: "input2"
+      )pb"));
   // Expect calculator to fail.
   EXPECT_FALSE(runner1.Run().ok());
 
-  CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
-    calculator: "MergeCalculator"
-    input_stream: "input1"
-    input_stream: "input2"
-    output_stream: "output1"
-    output_stream: "output2"
-  )"));
+  CalculatorRunner runner2(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
+        calculator: "MergeCalculator"
+        input_stream: "input1"
+        input_stream: "input2"
+        output_stream: "output1"
+        output_stream: "output2"
+      )pb"));
   // Expect calculator to fail.
   ASSERT_FALSE(runner2.Run().ok());
 }
@@ -58,12 +60,12 @@ TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
 // Ensures two streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingTwoStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integers 10, 20, 30, occurring at times 10, 20, 30.
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -102,13 +104,13 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
 // Ensures three streams with differing types can be merged correctly.
 TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
      TestMergingThreeStreams) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "MergeCalculator"
     input_stream: "input1"
     input_stream: "input2"
     input_stream: "input3"
     output_stream: "combined_output"
-  )"));
+  )pb"));
 
   // input1: integer 30 occurring at time 30.
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/core/mux_calculator_test.cc

@@ -31,7 +31,7 @@ namespace {
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input stream. All MuxCalculator inputs are present at each timestamp.
-constexpr char kTestGraphConfig1[] = R"proto(
+constexpr char kTestGraphConfig1[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -60,12 +60,12 @@ constexpr char kTestGraphConfig1[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with default input stream handler, and the input selection is driven
 // by an input side packet. All MuxCalculator inputs are present at each
 // timestamp.
-constexpr char kTestGraphConfig2[] = R"proto(
+constexpr char kTestGraphConfig2[] = R"pb(
   input_side_packet: "input_selector"
   input_stream: "input"
   output_stream: "test_output"
@@ -93,12 +93,12 @@ constexpr char kTestGraphConfig2[] = R"proto(
     output_stream: "OUTPUT:test_output"
     input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" }
   }
-)proto";
+)pb";
 
 // Graph with mux input stream handler, and the input selection is driven
 // by an input stream. Only one MuxCalculator input is present at each
 // timestamp.
-constexpr char kTestGraphConfig3[] = R"proto(
+constexpr char kTestGraphConfig3[] = R"pb(
   input_stream: "input"
   output_stream: "test_output"
   node {
@@ -117,7 +117,7 @@ constexpr char kTestGraphConfig3[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 constexpr char kOutputName[] = "test_output";
 constexpr char kInputName[] = "input";
@@ -235,7 +235,7 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
   EXPECT_EQ(output, input_packets);
 }
 
-constexpr char kDualInputGraphConfig[] = R"proto(
+constexpr char kDualInputGraphConfig[] = R"pb(
   input_stream: "input_0"
   input_stream: "input_1"
   input_stream: "input_select"
@@ -247,7 +247,7 @@ constexpr char kDualInputGraphConfig[] = R"proto(
     input_stream: "SELECT:input_select"
     output_stream: "OUTPUT:test_output"
   }
-)proto";
+)pb";
 
 TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
   CalculatorGraphConfig config =

mediapipe/calculators/core/non_zero_calculator.cc

@@ -0,0 +1,54 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
+class NonZeroCalculator : public Node {
+ public:
+  static constexpr Input<int>::SideFallback kIn{"INPUT"};
+  static constexpr Output<int>::Optional kOut{"OUTPUT"};
+  static constexpr Output<bool>::Optional kBooleanOut{"OUTPUT_BOOL"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kBooleanOut);
+
+  absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK(kOut(cc).IsConnected() || kBooleanOut(cc).IsConnected())
+        << "At least one output stream is expected.";
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      bool isNonZero = *kIn(cc) != 0;
+      if (kOut(cc).IsConnected()) {
+        kOut(cc).Send(std::make_unique<int>(isNonZero ? 1 : 0));
+      }
+      if (kBooleanOut(cc).IsConnected()) {
+        kBooleanOut(cc).Send(std::make_unique<bool>(isNonZero));
+      }
+    }
+    return absl::OkStatus();
+  }
+};
+
+MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/non_zero_calculator_test.cc

@@ -0,0 +1,93 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+class NonZeroCalculatorTest : public ::testing::Test {
+ protected:
+  NonZeroCalculatorTest()
+      : runner_(
+            R"pb(
+              calculator: "NonZeroCalculator"
+              input_stream: "INPUT:input"
+              output_stream: "OUTPUT:output"
+              output_stream: "OUTPUT_BOOL:output_bool"
+            )pb") {}
+
+  void SetInput(const std::vector<int>& inputs) {
+    int timestamp = 0;
+    for (const auto input : inputs) {
+      runner_.MutableInputs()
+          ->Get("INPUT", 0)
+          .packets.push_back(MakePacket<int>(input).At(Timestamp(timestamp++)));
+    }
+  }
+
+  std::vector<int> GetOutput() {
+    std::vector<int> result;
+    for (const auto output : runner_.Outputs().Get("OUTPUT", 0).packets) {
+      result.push_back(output.Get<int>());
+    }
+    return result;
+  }
+
+  std::vector<bool> GetOutputBool() {
+    std::vector<bool> result;
+    for (const auto output : runner_.Outputs().Get("OUTPUT_BOOL", 0).packets) {
+      result.push_back(output.Get<bool>());
+    }
+    return result;
+  }
+
+  CalculatorRunner runner_;
+};
+
+TEST_F(NonZeroCalculatorTest, ProducesZeroOutputForZeroInput) {
+  SetInput({0});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(0));
+  EXPECT_THAT(GetOutputBool(), ::testing::ElementsAre(false));
+}
+
+TEST_F(NonZeroCalculatorTest, ProducesNonZeroOutputForNonZeroInput) {
+  SetInput({1, 2, 3, -4, 5});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 1, 1, 1, 1));
+  EXPECT_THAT(GetOutputBool(),
+              ::testing::ElementsAre(true, true, true, true, true));
+}
+
+TEST_F(NonZeroCalculatorTest, SwitchesBetweenNonZeroAndZeroOutput) {
+  SetInput({1, 0, 3, 0, 5});
+  MP_ASSERT_OK(runner_.Run());
+  EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 0, 1, 0, 1));
+  EXPECT_THAT(GetOutputBool(),
+              ::testing::ElementsAre(true, false, true, false, true));
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_presence_calculator_test.cc

@@ -40,7 +40,7 @@ MATCHER_P2(BoolPacket, value, timestamp, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> output_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'allow'
         input_stream: 'value'
         node {
@@ -54,7 +54,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'PACKET:gated_value'
           output_stream: 'PRESENCE:presence'
         }
-      )");
+      )pb");
   tool::AddVectorSink("presence", &graph_config, &output_packets);
 
   CalculatorGraph graph;

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -87,7 +87,6 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
-  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -98,11 +97,7 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     output_data_id_ = cc->Outputs().GetId("", 0);
   }
 
-  period_count_ = 0;
   frame_rate_ = resampler_options.frame_rate();
-  base_timestamp_ = resampler_options.has_base_timestamp()
-                        ? Timestamp(resampler_options.base_timestamp())
-                        : Timestamp::Unset();
   start_time_ = resampler_options.has_start_time()
                     ? Timestamp(resampler_options.start_time())
                     : Timestamp::Min();
@@ -141,30 +136,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     }
   }
 
-  if (jitter_ != 0.0) {
-    if (resampler_options.output_header() !=
-        PacketResamplerCalculatorOptions::NONE) {
-      LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
-                      "the actual value.";
-    }
-    if (flush_last_packet_) {
-      flush_last_packet_ = false;
-      LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
-                      "ignored, because we are adding jitter.";
-    }
-    const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
-    random_ = CreateSecureRandom(seed);
-    if (random_ == nullptr) {
-      return absl::Status(
-          absl::StatusCode::kInvalidArgument,
-          "SecureRandom is not available.  With \"jitter\" specified, "
-          "PacketResamplerCalculator processing cannot proceed.");
-    }
-    packet_reservoir_random_ = CreateSecureRandom(seed);
-  }
-  packet_reservoir_ =
-      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
-  return absl::OkStatus();
+  strategy_ = GetSamplingStrategy(resampler_options);
+
+  return strategy_->Open(cc);
 }
 
 absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
@@ -177,171 +151,13 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
       return absl::OkStatus();
     }
   }
-  if (jitter_ != 0.0 && random_ != nullptr) {
-    // Packet reservior is used to make sure there's an output for every period,
-    // e.g. partial period at the end of the stream.
-    if (packet_reservoir_->IsEnabled() &&
-        (first_timestamp_ == Timestamp::Unset() ||
-         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
-      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
-      packet_reservoir_->AddSample(curr_packet);
-    }
-    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
-  } else {
-    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
+
+  if (absl::Status status = strategy_->Process(cc); !status.ok()) {
+    return status;  // Avoid MP_RETURN_IF_ERROR macro for external release.
   }
+
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
-  return absl::OkStatus();
-}
 
-void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
-  next_output_timestamp_min_ = first_timestamp_;
-  if (jitter_with_reflection_) {
-    next_output_timestamp_ =
-        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
-    return;
-  }
-  next_output_timestamp_ =
-      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
-}
-
-void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
-  packet_reservoir_->Clear();
-  if (jitter_with_reflection_) {
-    next_output_timestamp_min_ += frame_time_usec_;
-    Timestamp next_output_timestamp_max_ =
-        next_output_timestamp_min_ + frame_time_usec_;
-
-    next_output_timestamp_ += frame_time_usec_ +
-                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
-                              jitter_usec_;
-    next_output_timestamp_ = Timestamp(ReflectBetween(
-        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
-        next_output_timestamp_max_.Value()));
-    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
-    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
-    return;
-  }
-  packet_reservoir_->Disable();
-  next_output_timestamp_ +=
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_NE(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    first_timestamp_ = cc->InputTimestamp();
-    InitializeNextOutputTimestampWithJitter();
-    if (first_timestamp_ == next_output_timestamp_) {
-      OutputWithinLimits(
-          cc,
-          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
-      UpdateNextOutputTimestampWithJitter();
-    }
-    return absl::OkStatus();
-  }
-
-  if (frame_time_usec_ <
-      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
-    LOG_FIRST_N(WARNING, 2)
-        << "Adding jitter is not very useful when upsampling.";
-  }
-
-  while (true) {
-    const int64 last_diff =
-        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-    RET_CHECK_GT(last_diff, 0);
-    const int64 curr_diff =
-        (next_output_timestamp_ - cc->InputTimestamp()).Value();
-    if (curr_diff > 0) {
-      break;
-    }
-    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
-                                ? last_packet_
-                                : cc->Inputs().Get(input_data_id_).Value())
-                               .At(next_output_timestamp_));
-    UpdateNextOutputTimestampWithJitter();
-    // From now on every time a packet is emitted the timestamp of the next
-    // packet becomes known; that timestamp is stored in next_output_timestamp_.
-    // The only exception to this rule is the packet emitted from Close() which
-    // can only happen when jitter_with_reflection is enabled but in this case
-    // next_output_timestamp_min_ is a non-decreasing lower bound of any
-    // subsequent packet.
-    const Timestamp timestamp_bound = jitter_with_reflection_
-                                          ? next_output_timestamp_min_
-                                          : next_output_timestamp_;
-    cc->Outputs().Get(output_data_id_).SetNextTimestampBound(timestamp_bound);
-  }
-  return absl::OkStatus();
-}
-
-absl::Status PacketResamplerCalculator::ProcessWithoutJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_EQ(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    // This is the first packet, initialize the first_timestamp_.
-    if (base_timestamp_ == Timestamp::Unset()) {
-      // Initialize first_timestamp_ with exactly the first packet timestamp.
-      first_timestamp_ = cc->InputTimestamp();
-    } else {
-      // Initialize first_timestamp_ with the first packet timestamp
-      // aligned to the base_timestamp_.
-      int64 first_index = MathUtil::SafeRound<int64, double>(
-          (cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
-      first_timestamp_ =
-          base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
-    }
-    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
-      cc->Outputs()
-          .Tag("VIDEO_HEADER")
-          .Add(new VideoHeader(video_header_), Timestamp::PreStream());
-    }
-  }
-  const Timestamp received_timestamp = cc->InputTimestamp();
-  const int64 received_timestamp_idx =
-      TimestampToPeriodIndex(received_timestamp);
-  // Only consider the received packet if it belongs to the current period
-  // (== period_count_) or to a newer one (> period_count_).
-  if (received_timestamp_idx >= period_count_) {
-    // Fill the empty periods until we are in the same index as the received
-    // packet.
-    while (received_timestamp_idx > period_count_) {
-      OutputWithinLimits(
-          cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
-      ++period_count_;
-    }
-    // Now, if the received packet has a timestamp larger than the middle of
-    // the current period, we can send a packet without waiting. We send the
-    // one closer to the middle.
-    Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
-    if (received_timestamp >= target_timestamp) {
-      bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
-      bool send_current =
-          !have_last_packet || (received_timestamp - target_timestamp <=
-                                target_timestamp - last_packet_.Timestamp());
-      if (send_current) {
-        OutputWithinLimits(
-            cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
-      } else {
-        OutputWithinLimits(cc, last_packet_.At(target_timestamp));
-      }
-      ++period_count_;
-    }
-    // TODO: Add a mechanism to the framework to allow these packets
-    // to be output earlier (without waiting for a much later packet to
-    // arrive)
-
-    // Update the bound for the next packet.
-    cc->Outputs()
-        .Get(output_data_id_)
-        .SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
-  }
   return absl::OkStatus();
 }
 
@@ -349,17 +165,34 @@ absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
   if (!cc->GraphStatus().ok()) {
     return absl::OkStatus();
   }
-  // Emit the last packet received if we have at least one packet, but
-  // haven't sent anything for its period.
-  if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
-      TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
-    OutputWithinLimits(cc,
-                       last_packet_.At(PeriodIndexToTimestamp(period_count_)));
+
+  return strategy_->Close(cc);
+}
+
+std::unique_ptr<PacketResamplerStrategy>
+PacketResamplerCalculator::GetSamplingStrategy(
+    const PacketResamplerCalculatorOptions& options) {
+  if (options.reproducible_sampling()) {
+    if (!options.jitter_with_reflection()) {
+      LOG(WARNING)
+          << "reproducible_sampling enabled w/ jitter_with_reflection "
+             "disabled. "
+          << "reproducible_sampling always uses jitter with reflection, "
+          << "Ignoring jitter_with_reflection setting.";
+    }
+    return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
   }
-  if (!packet_reservoir_->IsEmpty()) {
-    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+
+  if (options.jitter() == 0) {
+    return absl::make_unique<NoJitterStrategy>(this);
   }
-  return absl::OkStatus();
+
+  if (options.jitter_with_reflection()) {
+    return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
+  }
+
+  // With jitter and no reflection.
+  return absl::make_unique<JitterWithoutReflectionStrategy>(this);
 }
 
 Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
@@ -385,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
   }
 }
 
+absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    LOG(INFO) << "Emitting pack from reservoir.";
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (packet_reservoir_->IsEnabled() &&
+      (first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (first_timestamp_ == Timestamp::Unset()) {
+    first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestampWithJitter();
+    if (first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
+    // From now on every time a packet is emitted the timestamp of the next
+    // packet becomes known; that timestamp is stored in next_output_timestamp_.
+    // The only exception to this rule is the packet emitted from Close() which
+    // can only happen when jitter_with_reflection is enabled but in this case
+    // next_output_timestamp_min_ is a non-decreasing lower bound of any
+    // subsequent packet.
+    const Timestamp timestamp_bound = next_output_timestamp_min_;
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(timestamp_bound);
+  }
+  return absl::OkStatus();
+}
+
+void LegacyJitterWithReflectionStrategy::
+    InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
+  next_output_timestamp_ =
+      first_timestamp_ +
+      random_->UnbiasedUniform64(calculator_->frame_time_usec_);
+}
+
+void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  next_output_timestamp_min_ += calculator_->frame_time_usec_;
+  Timestamp next_output_timestamp_max_ =
+      next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+  next_output_timestamp_ +=
+      calculator_->frame_time_usec_ +
+      random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
+      calculator_->jitter_usec_;
+  next_output_timestamp_ = Timestamp(ReflectBetween(
+      next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+      next_output_timestamp_max_.Value()));
+  CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+  CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+}
+
+absl::Status ReproducibleJitterWithReflectionStrategy::Open(
+    CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Close(
+    CalculatorContext* cc) {
+  // If last packet is non-empty and a packet hasn't been emitted for this
+  // period, emit the last packet.
+  if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(next_output_timestamp_));
+  }
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+
+  if (calculator_->last_packet_.IsEmpty()) {
+    // last_packet is empty, this is the first packet of the stream.
+
+    InitializeNextOutputTimestamp(current_packet.Timestamp());
+
+    // If next_output_timestamp_ happens to fall before current_packet, emit
+    // current packet.  Only a single packet can be emitted at the beginning
+    // of the stream.
+    if (next_output_timestamp_ < current_packet.Timestamp()) {
+      calculator_->OutputWithinLimits(
+          cc, current_packet.At(next_output_timestamp_));
+      packet_emitted_this_period_ = true;
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Last packet is set, so we are mid-stream.
+  if (calculator_->frame_time_usec_ <
+      (current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
+          .Value()) {
+    // Note, if the stream is upsampling, this could lead to the same packet
+    // being emitted twice.  Upsampling and jitter doesn't make much sense
+    // but does technically work.
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  // Since we may be upsampling, we need to iteratively advance the
+  // next_output_timestamp_ one period at a time until it reaches the period
+  // current_packet is in.  During this process, last_packet and/or
+  // current_packet may be repeatly emitted.
+
+  UpdateNextOutputTimestamp(current_packet.Timestamp());
+
+  while (!packet_emitted_this_period_ &&
+         next_output_timestamp_ <= current_packet.Timestamp()) {
+    // last_packet < next_output_timestamp_ <= current_packet,
+    // so emit the closest packet.
+    Packet packet_to_emit =
+        current_packet.Timestamp() - next_output_timestamp_ <
+                next_output_timestamp_ - calculator_->last_packet_.Timestamp()
+            ? current_packet
+            : calculator_->last_packet_;
+    calculator_->OutputWithinLimits(cc,
+                                    packet_to_emit.At(next_output_timestamp_));
+
+    packet_emitted_this_period_ = true;
+
+    // If we are upsampling, packet_emitted_this_period_ can be reset by
+    // the following UpdateNext and the loop will iterate.
+    UpdateNextOutputTimestamp(current_packet.Timestamp());
+  }
+
+  // Set the bounds on the output stream.  Note, if we emitted a packet
+  // above, it will already be set at next_output_timestamp_ + 1, in which
+  // case we have to skip setting it.
+  if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
+      next_output_timestamp_) {
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (next_output_timestamp_min_ != Timestamp::Unset()) {
+    return;
+  }
+
+  next_output_timestamp_min_ = Timestamp(0);
+  next_output_timestamp_ =
+      Timestamp(GetNextRandom(calculator_->frame_time_usec_));
+
+  // While the current timestamp is ahead of the max (i.e. min + frame_time),
+  // fast-forward.
+  while (current_timestamp >=
+         next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    packet_emitted_this_period_ = true;  // Force update...
+    UpdateNextOutputTimestamp(current_timestamp);
+  }
+}
+
+void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (packet_emitted_this_period_ &&
+      current_timestamp >=
+          next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    next_output_timestamp_min_ += calculator_->frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+    next_output_timestamp_ += calculator_->frame_time_usec_ +
+                              GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
+                              calculator_->jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+
+    packet_emitted_this_period_ = false;
+  }
+}
+
+absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  // Packet reservior is used to make sure there's an output for every period,
+  // e.g. partial period at the end of the stream.
+  if (packet_reservoir_->IsEnabled() &&
+      (calculator_->first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    calculator_->first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestamp();
+    if (calculator_->first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestamp();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestamp();
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
+  next_output_timestamp_min_ = calculator_->first_timestamp_;
+  next_output_timestamp_ = calculator_->first_timestamp_ +
+                           calculator_->frame_time_usec_ * random_->RandFloat();
+}
+
+void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
+  packet_reservoir_->Clear();
+  packet_reservoir_->Disable();
+  next_output_timestamp_ += calculator_->frame_time_usec_ *
+                            ((1.0 - calculator_->jitter_) +
+                             2.0 * calculator_->jitter_ * random_->RandFloat());
+}
+
+absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+  base_timestamp_ = resampler_options.has_base_timestamp()
+                        ? Timestamp(resampler_options.base_timestamp())
+                        : Timestamp::Unset();
+
+  period_count_ = 0;
+
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
+  // Emit the last packet received if we have at least one packet, but
+  // haven't sent anything for its period.
+  if (calculator_->first_timestamp_ != Timestamp::Unset() &&
+      calculator_->flush_last_packet_ &&
+      calculator_->TimestampToPeriodIndex(
+          calculator_->last_packet_.Timestamp()) == period_count_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(
+                calculator_->PeriodIndexToTimestamp(period_count_)));
+  }
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    // This is the first packet, initialize the first_timestamp_.
+    if (base_timestamp_ == Timestamp::Unset()) {
+      // Initialize first_timestamp_ with exactly the first packet timestamp.
+      calculator_->first_timestamp_ = cc->InputTimestamp();
+    } else {
+      // Initialize first_timestamp_ with the first packet timestamp
+      // aligned to the base_timestamp_.
+      int64 first_index = MathUtil::SafeRound<int64, double>(
+          (cc->InputTimestamp() - base_timestamp_).Seconds() *
+          calculator_->frame_rate_);
+      calculator_->first_timestamp_ =
+          base_timestamp_ +
+          TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
+    }
+    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
+      cc->Outputs()
+          .Tag("VIDEO_HEADER")
+          .Add(new VideoHeader(calculator_->video_header_),
+               Timestamp::PreStream());
+    }
+  }
+  const Timestamp received_timestamp = cc->InputTimestamp();
+  const int64 received_timestamp_idx =
+      calculator_->TimestampToPeriodIndex(received_timestamp);
+  // Only consider the received packet if it belongs to the current period
+  // (== period_count_) or to a newer one (> period_count_).
+  if (received_timestamp_idx >= period_count_) {
+    // Fill the empty periods until we are in the same index as the received
+    // packet.
+    while (received_timestamp_idx > period_count_) {
+      calculator_->OutputWithinLimits(
+          cc, calculator_->last_packet_.At(
+                  calculator_->PeriodIndexToTimestamp(period_count_)));
+      ++period_count_;
+    }
+    // Now, if the received packet has a timestamp larger than the middle of
+    // the current period, we can send a packet without waiting. We send the
+    // one closer to the middle.
+    Timestamp target_timestamp =
+        calculator_->PeriodIndexToTimestamp(period_count_);
+    if (received_timestamp >= target_timestamp) {
+      bool have_last_packet =
+          (calculator_->last_packet_.Timestamp() != Timestamp::Unset());
+      bool send_current =
+          !have_last_packet ||
+          (received_timestamp - target_timestamp <=
+           target_timestamp - calculator_->last_packet_.Timestamp());
+      if (send_current) {
+        calculator_->OutputWithinLimits(cc,
+                                        cc->Inputs()
+                                            .Get(calculator_->input_data_id_)
+                                            .Value()
+                                            .At(target_timestamp));
+      } else {
+        calculator_->OutputWithinLimits(
+            cc, calculator_->last_packet_.At(target_timestamp));
+      }
+      ++period_count_;
+    }
+    // TODO: Add a mechanism to the framework to allow these packets
+    // to be output earlier (without waiting for a much later packet to
+    // arrive)
+
+    // Update the bound for the next packet.
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(
+            calculator_->PeriodIndexToTimestamp(period_count_));
+  }
+  return absl::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -55,7 +55,7 @@ class PacketReservoir {
 //     correspond to timestamp t.
 //   - The next packet is chosen randomly (uniform distribution) among frames
 //     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
-//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//     - if jitter_with_reflection is true, the timestamp will be reflected
 //       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
 //       so that its marginal distribution is uniform within this interval.
 //       In the formula, t_0 is the timestamp of the first sampled
@@ -66,6 +66,17 @@ class PacketReservoir {
 //     the resampling.  For Cloud ML Video Intelligence API, the hash of the
 //     input video should serve this purpose.  For YouTube, either video ID or
 //     content hex ID of the input video should do.
+//   - If reproducible_samping is true, care is taken to allow reproducible
+//     "mid-stream" sampling.  The calculator can be executed on a stream that
+//     doesn't start at the first period.  For instance, if the calculator
+//     is run on a 10 second stream it will produce the same set of samples
+//     as two runs of the calculator, the first with 3 seconds of input starting
+//     at time 0 and the second with 7 seconds of input starting at time +3s.
+//     - In order to guarantee the exact same samples, 1) the inputs must be
+//       aligned with the sampling period.  For instance, if the sampling rate
+//       is 2 frames per second, streams should be aligned on 0.5 second
+//       boundaries, and 2) the stream must include at least one extra packet
+//       before and after the second aligned sampling period.
 //
 // If jitter_ is not specified:
 //   - The first packet defines the first_timestamp of the output stream,
@@ -105,19 +116,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
   absl::Status Process(CalculatorContext* cc) override;
 
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  absl::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  absl::Status ProcessWithoutJitter(CalculatorContext* cc);
-
   // Given the current count of periods that have passed, this returns
   // the next valid timestamp of the middle point of the next period:
   //    if count is 0, it returns the first_timestamp_.
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Outputs a packet if it is in range (start_time_, end_time_).
   void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
 
+ protected:
+  // Returns Sampling Strategy to use.
+  //
+  // Virtual to allow injection of testing strategies.
+  virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& options);
+
+ private:
+  std::unique_ptr<class PacketResamplerStrategy> strategy_;
+
   // The timestamp of the first packet received.
   Timestamp first_timestamp_;
 
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Inverse of frame_rate_.
   int64 frame_time_usec_;
 
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
   VideoHeader video_header_;
   // The "DATA" input stream.
   CollectionItemId input_data_id_;
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
   CollectionItemId output_data_id_;
 
   // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  // than the final stream timestamp.
   bool flush_last_packet_;
 
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
   double jitter_ = 0.0;
-  bool jitter_with_reflection_;
-  int64 jitter_usec_;
-  Timestamp next_output_timestamp_;
-  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
-  // kept within the interval
-  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
-  Timestamp next_output_timestamp_min_;
 
-  // If specified, output timestamps are aligned with base_timestamp.
-  // Otherwise, they are aligned with the first input timestamp.
-  Timestamp base_timestamp_;
+  int64 jitter_usec_;
+
+  // The last packet that was received.
+  Packet last_packet_;
 
   // If specified, only outputs at/after start_time are included.
   Timestamp start_time_;
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
 
   // If set, the output timestamps nearest to start_time and end_time
   // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.W
+  // between start_time and end_time.
   bool round_limits_;
 
+  // Allow strategies access to all internal calculator state.
+  //
+  // The calculator and strategies are intimiately tied together so this should
+  // not break encapsulation.
+  friend class LegacyJitterWithReflectionStrategy;
+  friend class ReproducibleJitterWithReflectionStrategy;
+  friend class JitterWithoutReflectionStrategy;
+  friend class NoJitterStrategy;
+};
+
+// Abstract class encapsulating sampling stategy.
+//
+// These are used solely by PacketResamplerCalculator, but are exposed here
+// to facilitate tests.
+class PacketResamplerStrategy {
+ public:
+  PacketResamplerStrategy(PacketResamplerCalculator* calculator)
+      : calculator_(calculator) {}
+  virtual ~PacketResamplerStrategy() = default;
+
+  // Delegate for CalculatorBase::Open.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Open(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Close.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Close(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Process.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Process(CalculatorContext* cc) = 0;
+
+ protected:
+  // Calculator running strategy.
+  PacketResamplerCalculator* calculator_;
+};
+
+// Strategy that applies Jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+//
+// This applies the legacy jitter with reflection which doesn't allow
+// for reproducibility of sampling when starting mid-stream.  This is maintained
+// for backward compatibility.
+class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void InitializeNextOutputTimestampWithJitter();
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
   // packet reservior used for sampling random packet out of partial
   // period when jitter is enabled
   std::unique_ptr<PacketReservoir> packet_reservoir_;
+
   // random number generator used in packet_reservior_.
   std::unique_ptr<RandomBase> packet_reservoir_random_;
 };
 
+// Strategy that applies reproducible jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+class ReproducibleJitterWithReflectionStrategy
+    : public PacketResamplerStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategy(
+      PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ protected:
+  // Returns next random in range (0,n].
+  //
+  // Exposed as virtual function for testing Jitter with reflection.
+  // This is the only way random_ is accessed.
+  virtual uint64 GetNextRandom(uint64 n) {
+    return random_->UnbiasedUniform64(n);
+  }
+
+ private:
+  // Initializes Jitter with reflection.
+  //
+  // This will fast-forward to the period containing current_timestamp.
+  // next_output_timestamp_ is guarnateed to be current_timestamp's period
+  // and packet_emitted_this_period_ will be set to false.
+  void InitializeNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Potentially advances next_output_timestamp_ a single period.
+  //
+  // next_output_timestamp_ will only be advanced if packet_emitted_this_period_
+  // is false.  next_output_timestamp_ will never be advanced beyond
+  // current_timestamp's period.
+  //
+  // However, next_output_timestamp_ could fall before current_timestamp's
+  // period since only a single period can be advanced at a time.
+  void UpdateNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // Indicates packet was emitted for current period (i.e. the period
+  // next_output_timestamp_ falls in.
+  bool packet_emitted_this_period_ = false;
+};
+
+// Strategy that applies Jitter without reflection based sampling.
+//
+// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
+// not enabled.
+class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestamp();
+
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestamp();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // packet reservior used for sampling random packet out of partial period.
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+// Strategy that applies sampling without any jitter.
+//
+// Used by PacketResamplerCalculator when jitter is not enabled.
+class NoJitterStrategy : public PacketResamplerStrategy {
+ public:
+  NoJitterStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Number of periods that have passed (= #packets sent to the output).
+  int64 period_count_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+};
+
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+
+  // Enables reflection when applying jitter.
+  //
+  // This option is ignored when reproducible_sampling is true, in which case
+  // reflection will be used.
+  //
+  // New use cases should use reproducible_sampling = true, as
+  // jitter_with_reflection is deprecated and will be removed at some point.
   optional bool jitter_with_reflection = 9 [default = false];
 
+  // If set, enabled reproducible sampling, allowing frames to be sampled
+  // without regards to where the stream starts.  See
+  // packet_resampler_calculator.h for details.
+  //
+  // This enables reflection (ignoring jitter_with_reflection setting).
+  optional bool reproducible_sampling = 10 [default = false];
+
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.
   //

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -30,6 +30,7 @@
 
 namespace mediapipe {
 
+using ::testing::ElementsAre;
 namespace {
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
   static int static_count_;
 };
 
+// Matcher for Packets with uint64 payload, comparing arg packet's
+// timestamp and uint64 payload.
+MATCHER_P2(PacketAtTimestamp, payload, timestamp,
+           absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
+                        payload, " @ time ", timestamp)) {
+  if (timestamp != arg.Timestamp().Value()) {
+    *result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
+    return false;
+  }
+  int64 actual_payload = arg.template Get<int64>();
+  if (actual_payload != payload) {
+    *result_listener << "with incorrect payload = " << actual_payload;
+    return false;
+  }
+  return true;
+}
+
+// JitterWithReflectionStrategy child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleJitterWithReflectionStrategyForTesting
+    : public ReproducibleJitterWithReflectionStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategyForTesting(
+      PacketResamplerCalculator* calculator)
+      : ReproducibleJitterWithReflectionStrategy(calculator) {}
+
+  // Statically accessed random sequence to use for jitter with reflection.
+  //
+  // An EXPECT will fail if sequence is less than the number requested during
+  // processing.
+  static std::vector<uint64> random_sequence;
+
+ protected:
+  virtual uint64 GetNextRandom(uint64 n) {
+    EXPECT_LT(sequence_index_, random_sequence.size());
+    return random_sequence[sequence_index_++] % n;
+  }
+
+ private:
+  int32 sequence_index_ = 0;
+};
+std::vector<uint64>
+    ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
+
+// PacketResamplerCalculator child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleResamplerCalculatorForTesting
+    : public PacketResamplerCalculator {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    return PacketResamplerCalculator::GetContract(cc);
+  }
+
+ protected:
+  std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& Options) {
+    return absl::make_unique<
+        ReproducibleJitterWithReflectionStrategyForTesting>(this);
+  }
+};
+
+REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
+
 int SimpleRunner::static_count_ = 0;
 
 TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
@@ -380,7 +452,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
 }
 
 TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
-  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "PacketResamplerCalculator"
     input_stream: "DATA:in_data"
     input_stream: "VIDEO_HEADER:in_video_header"
@@ -389,7 +461,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
     options {
       [mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 }
     }
-  )"));
+  )pb"));
 
   for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
     runner.MutableInputs()->Tag("DATA").packets.push_back(
@@ -633,7 +705,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
 
 TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "PacketResamplerCalculator"
         input_side_packet: "OPTIONS:options"
         input_stream: "input"
@@ -643,16 +715,16 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
             frame_rate: 60
             base_timestamp: 0
           }
-        })");
+        })pb");
 
   {
     SimpleRunner runner(node_config);
     auto options =
         new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(
-            R"(
+            R"pb(
               [mediapipe.PacketResamplerCalculatorOptions.ext] {
                 frame_rate: 30
-              })"));
+              })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
     MP_ASSERT_OK(runner.Run());
@@ -662,12 +734,12 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
     SimpleRunner runner(node_config);
 
     auto options =
-        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"(
+        new CalculatorOptions(ParseTextProtoOrDie<CalculatorOptions>(R"pb(
           merge_fields: false
           [mediapipe.PacketResamplerCalculatorOptions.ext] {
             frame_rate: 30
             base_timestamp: 0
-          })"));
+          })pb"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -69,7 +69,7 @@ MATCHER_P2(PairPacket, timestamp, pair, "") {
 TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   std::vector<Packet> in_prev;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -93,7 +93,7 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
           input_stream: 'previous2'
           output_stream: 'pair'
         }
-      )");
+      )pb");
   tool::AddVectorSink("pair", &graph_config_, &in_prev);
 
   CalculatorGraph graph_;
@@ -169,7 +169,7 @@ REGISTER_CALCULATOR(PacketOnCloseCalculator);
 TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -192,7 +192,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -231,7 +231,7 @@ TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -253,7 +253,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -278,7 +278,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) {
 TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
   std::vector<Packet> out_and_previous_packets;
   CalculatorGraphConfig graph_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -300,7 +300,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
           input_stream: 'previous'
           output_stream: 'out_and_previous'
         }
-      )");
+      )pb");
   tool::AddVectorSink("out_and_previous", &graph_config,
                       &out_and_previous_packets);
 
@@ -331,7 +331,7 @@ TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) {
 TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
   std::vector<Packet> outputs;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         node {
           calculator: 'PreviousLoopbackCalculator'
@@ -354,7 +354,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
           input_stream: 'out'
           output_stream: 'close_out'
         }
-      )");
+      )pb");
   tool::AddVectorSink("close_out", &graph_config_, &outputs);
 
   CalculatorGraph graph_;
@@ -386,7 +386,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           input_stream: 'force_main_empty'
           input_stream: 'force_loop_empty'
@@ -424,7 +424,7 @@ class PreviousLoopbackCalculatorProcessingTimestampsTest
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));
@@ -724,7 +724,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
  protected:
   void SetUp() override {
     CalculatorGraphConfig graph_config =
-        ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
           input_stream: 'input'
           # Drops "loop" when set to "true", delaying output of prev_loop, hence
           # delaying output of the graph.
@@ -755,7 +755,7 @@ class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test {
             input_stream: 'passed_through_prev_loop'
             output_stream: 'passed_through_input_and_prev_loop'
           }
-        )");
+        )pb");
     tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config,
                         &output_packets_);
     MP_ASSERT_OK(graph_.Initialize(graph_config, {}));

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -27,7 +27,7 @@ namespace mediapipe {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -36,7 +36,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -53,7 +53,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -63,7 +63,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -80,7 +80,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
 
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -90,7 +90,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
             min_quantized_value: 1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -107,7 +107,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -117,7 +117,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
             min_quantized_value: -1
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
@@ -133,7 +133,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -143,7 +143,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
   runner.MutableInputs()
@@ -171,7 +171,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
 
 TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "QuantizeFloatVectorCalculator"
         input_stream: "FLOAT_VECTOR:float_vector"
         output_stream: "ENCODED:encoded"
@@ -181,7 +181,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
             min_quantized_value: -64
           }
         }
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {-65.0f, 65.0f};
   runner.MutableInputs()

mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc

@@ -70,13 +70,13 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
 constexpr int kNumImageFrames = 5;
 constexpr int kNumFinished = 3;
 CalculatorGraphConfig::Node GetDefaultNode() {
-  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+  return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
     calculator: "RealTimeFlowLimiterCalculator"
     input_stream: "raw_frames"
     input_stream: "FINISHED:finished"
     input_stream_info: { tag_index: "FINISHED" back_edge: true }
     output_stream: "gated_frames"
-  )");
+  )pb");
 }
 
 // Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just
@@ -219,7 +219,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
   // Back-edge "finished" limits processing to one frame in-flight.
   // The two LambdaCalculators are used to keep certain packet sets in flight.
   CalculatorGraphConfig InflightGraphConfig() {
-    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    return ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
       input_stream: 'in_1'
       input_stream: 'in_2'
       node {
@@ -256,7 +256,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
         output_stream: 'out_1'
         output_stream: 'out_2'
       }
-    )");
+    )pb");
   }
 
  protected:
@@ -344,7 +344,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
   std::vector<Packet> a_passed;
   std::vector<Packet> b_passed;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in_a'
         input_stream: 'in_b'
         input_stream: 'finished'
@@ -360,7 +360,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
           output_stream: 'in_b_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed);
   tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed);
@@ -442,7 +442,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
 TEST(RealTimeFlowLimiterCalculator, CanConsume) {
   std::vector<Packet> in_sampled_packets_;
   CalculatorGraphConfig graph_config_ =
-      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
         input_stream: 'in'
         input_stream: 'finished'
         node {
@@ -455,7 +455,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
           output_stream: 'in_sampled'
           output_stream: 'ALLOW:allow'
         }
-      )");
+      )pb");
   std::string allow_cb_name;
   tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_);
   tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true);

mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc

@@ -36,7 +36,7 @@ using testing::HasSubstr;
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -45,7 +45,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -58,7 +58,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -67,7 +67,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -79,7 +79,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -88,7 +88,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
               input_side_packet: "side_packet"
               output_stream: "DOES_NOT_EXIST:packet"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -102,7 +102,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
 TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -113,7 +113,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
               output_stream: "AT_TICK:packet0"
               output_stream: "AT_PRE_STREAM:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -127,7 +127,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -136,7 +136,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
               output_stream: "AT_PRESTREAM:0:packet0"
               output_stream: "AT_PRESTREAM:1:packet1"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -149,7 +149,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
 TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
             node {
@@ -158,7 +158,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
               input_side_packet: "side_packet1"
               output_stream: "AT_PRESTREAM:packet0"
             }
-          )");
+          )pb");
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
@@ -209,7 +209,7 @@ TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
 TEST(SidePacketToStreamCalculator, AtTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -219,7 +219,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
               input_side_packet: "side_packet"
               output_stream: "AT_TICK:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -251,7 +251,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
 TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tick"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -265,7 +265,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
               output_stream: "AT_TICK:0:packet0"
               output_stream: "AT_TICK:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;
@@ -305,7 +305,7 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
 TEST(SidePacketToStreamCalculator, AtTimestamp) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet"
             output_stream: "packet"
@@ -315,7 +315,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
               input_side_packet: "side_packet"
               output_stream: "AT_TIMESTAMP:packet"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets;
   tool::AddVectorSink("packet", &graph_config, &output_packets);
   CalculatorGraph graph;
@@ -337,7 +337,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
 TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_side_packet: "timestamp"
             input_side_packet: "side_packet0"
             input_side_packet: "side_packet1"
@@ -350,7 +350,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
               output_stream: "AT_TIMESTAMP:0:packet0"
               output_stream: "AT_TIMESTAMP:1:packet1"
             }
-          )");
+          )pb");
   std::vector<Packet> output_packets0;
   tool::AddVectorSink("packet0", &graph_config, &output_packets0);
   std::vector<Packet> output_packets1;

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -122,7 +122,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -138,7 +138,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -171,7 +171,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -183,7 +183,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -196,7 +196,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -209,7 +209,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -223,7 +223,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -238,7 +238,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -252,7 +252,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -266,7 +266,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -281,7 +281,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -298,7 +298,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -334,7 +334,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -349,7 +349,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -377,7 +377,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
   // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "landmarks_in"
             node {
               calculator: "SplitNormalizedLandmarkListCalculator"
@@ -394,7 +394,7 @@ TEST_F(SplitNormalizedLandmarkListCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -163,7 +163,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -179,7 +179,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -214,7 +214,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -226,7 +226,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -240,7 +240,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -253,7 +253,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -269,7 +269,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -284,7 +284,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -299,7 +299,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -313,7 +313,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -330,7 +330,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -347,7 +347,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
   std::vector<Packet> range_1_packets;
@@ -385,7 +385,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -400,7 +400,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
 
@@ -428,7 +428,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
   // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "tensor_in"
             node {
               calculator: "SplitTfLiteTensorVectorCalculator"
@@ -445,7 +445,7 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest,
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -511,7 +511,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -524,7 +524,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   // Run the graph.
   CalculatorGraph graph;
@@ -536,7 +536,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -552,7 +552,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -592,7 +592,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -609,7 +609,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
@@ -646,7 +646,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
   // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator.
   CalculatorGraphConfig graph_config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
+          R"pb(
             input_stream: "input_vector"
             node {
               calculator: "MovableSplitUniqueIntPtrCalculator"
@@ -661,7 +661,7 @@ TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) {
                 }
               }
             }
-          )");
+          )pb");
 
   std::vector<Packet> range_0_packets;
   tool::AddVectorSink("range_0", &graph_config, &range_0_packets);

mediapipe/calculators/image/BUILD

@@ -405,12 +405,44 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "image_clone_calculator_proto",
+    srcs = ["image_clone_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "image_clone_calculator",
+    srcs = ["image_clone_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":image_clone_calculator_cc_proto",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gl_calculator_helper",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "image_properties_calculator",
     srcs = ["image_properties_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",

mediapipe/calculators/image/image_clone_calculator.cc

@@ -0,0 +1,125 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/image_clone_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/port/status.h"
+
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+namespace api2 {
+
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+using GpuBuffer = AnyType;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
+
+// Clones an input image and makes sure in the output clone the pixel data are
+// stored on the target storage (CPU vs GPU) specified in the calculator option.
+//
+// The clone shares ownership of the input pixel data on the existing storage.
+// If the target storage is diffrent from the existing one, then the data is
+// further copied there.
+//
+// Example usage:
+// node {
+//   calculator: "ImageCloneCalculator"
+//   input_stream: "input"
+//   output_stream: "output"
+//   options: {
+//     [mediapipe.ImageCloneCalculatorOptions.ext] {
+//       output_on_gpu: true
+//     }
+//   }
+// }
+class ImageCloneCalculator : public Node {
+ public:
+  static constexpr Input<Image> kIn{""};
+  static constexpr Output<Image> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+#if MEDIAPIPE_DISABLE_GPU
+    if (cc->Options<mediapipe::ImageCloneCalculatorOptions>().output_on_gpu()) {
+      return absl::UnimplementedError(
+          "GPU processing is disabled in build flags");
+    }
+#else
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  // MEDIAPIPE_DISABLE_GPU
+    return absl::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    const auto& options = cc->Options<mediapipe::ImageCloneCalculatorOptions>();
+    output_on_gpu_ = options.output_on_gpu();
+#if !MEDIAPIPE_DISABLE_GPU
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+#endif  // !MEDIAPIPE_DISABLE_GPU
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    std::unique_ptr<Image> output;
+    const auto& input = *kIn(cc);
+    if (input.UsesGpu()) {
+#if !MEDIAPIPE_DISABLE_GPU
+      // Create an output Image that co-owns the underlying texture buffer as
+      // the input Image.
+      output = std::make_unique<Image>(input.GetGpuBuffer());
+#endif  // !MEDIAPIPE_DISABLE_GPU
+    } else {
+      // Make a copy of the input packet to co-own the input Image.
+      mediapipe::Packet* packet_copy_ptr =
+          new mediapipe::Packet(kIn(cc).packet());
+      // Create an output Image that (co-)owns a new ImageFrame that points to
+      // the same pixel data as the input Image and also owns the packet
+      // copy. As a result, the output Image indirectly co-owns the input
+      // Image. This ensures a correct life span of the shared pixel data.
+      output = std::make_unique<Image>(std::make_unique<mediapipe::ImageFrame>(
+          input.image_format(), input.width(), input.height(), input.step(),
+          const_cast<uint8*>(input.GetImageFrameSharedPtr()->PixelData()),
+          [packet_copy_ptr](uint8*) { delete packet_copy_ptr; }));
+    }
+
+    if (output_on_gpu_) {
+#if !MEDIAPIPE_DISABLE_GPU
+      gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); });
+#endif  // !MEDIAPIPE_DISABLE_GPU
+    } else {
+      output->ConvertToCpu();
+    }
+    kOut(cc).Send(std::move(output));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  bool output_on_gpu_;
+#if !MEDIAPIPE_DISABLE_GPU
+  mediapipe::GlCalculatorHelper gpu_helper_;
+#endif  // !MEDIAPIPE_DISABLE_GPU
+};
+MEDIAPIPE_REGISTER_NODE(ImageCloneCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/image/image_clone_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ImageCloneCalculatorOptions {
+  extend CalculatorOptions {
+    optional ImageCloneCalculatorOptions ext = 372781894;
+  }
+
+  // Whether the output clone should have pixel data already available on GPU.
+  optional bool output_on_gpu = 1 [default = false];
+}

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -285,7 +285,7 @@ absl::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
 
   // Run cropping shader on GPU.
   {
-    gpu_helper_.BindFramebuffer(dst_tex);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(dst_tex);
 
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(src_tex.target(), src_tex.name());

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -41,7 +41,7 @@ constexpr char kWidthTag[] = "WIDTH";
 TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -54,7 +54,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -79,7 +79,7 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             output_stream: "IMAGE_GPU:cropped_output_frames"
@@ -94,7 +94,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -119,7 +119,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
 TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "WIDTH:crop_width"
@@ -136,7 +136,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -168,7 +168,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
 TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
   auto calculator_node =
       ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig::Node>(
-          R"(
+          R"pb(
             calculator: "ImageCroppingCalculator"
             input_stream: "IMAGE_GPU:input_frames"
             input_stream: "RECT:rect"
@@ -184,7 +184,7 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
                 rotation: 0.3
               }
             }
-          )");
+          )pb");
 
   auto calculator_state = absl::make_unique<CalculatorState>(
       "Node", 0, "Calculator", calculator_node, nullptr);
@@ -196,9 +196,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
       calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
   auto& inputs = cc->Inputs();
   mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
-      R"(
+      R"pb(
         width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
-      )");
+      )pb");
   inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
   RectSpec expectRect = {
       .width = 1,

mediapipe/calculators/image/image_file_properties_calculator_test.cc

@@ -50,11 +50,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_stream: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
@@ -79,11 +79,11 @@ TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) {
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_side_packet: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Index(0) =
@@ -108,11 +108,11 @@ TEST(ImageFilePropertiesCalculatorTest,
   MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "ImageFilePropertiesCalculator"
         input_stream: "image_bytes"
         output_side_packet: "properties"
-      )");
+      )pb");
 
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(

mediapipe/calculators/image/image_properties_calculator.cc

@@ -12,25 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/image_frame.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-namespace {
-constexpr char kImageFrameTag[] = "IMAGE";
-constexpr char kGpuBufferTag[] = "IMAGE_GPU";
-}  // namespace
-
 namespace mediapipe {
+namespace api2 {
+
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+using GpuBuffer = AnyType;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
 
 // Extracts image properties from the input image and outputs the properties.
 // Currently only supports image size.
 // Input:
 //   One of the following:
-//   IMAGE: An ImageFrame
+//   IMAGE: An Image or ImageFrame (for backward compatibility with existing
+//          graphs that use IMAGE for ImageFrame input)
+//   IMAGE_CPU: An ImageFrame
 //   IMAGE_GPU: A GpuBuffer
 //
 // Output:
@@ -42,59 +49,64 @@ namespace mediapipe {
 //   input_stream: "IMAGE:image"
 //   output_stream: "SIZE:size"
 // }
-class ImagePropertiesCalculator : public CalculatorBase {
+class ImagePropertiesCalculator : public Node {
  public:
-  static absl::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-              cc->Inputs().HasTag(kGpuBufferTag));
-    if (cc->Inputs().HasTag(kImageFrameTag)) {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    }
-#if !MEDIAPIPE_DISABLE_GPU
-    if (cc->Inputs().HasTag(kGpuBufferTag)) {
-      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
-    }
-#endif  // !MEDIAPIPE_DISABLE_GPU
+  static constexpr Input<
+      OneOf<mediapipe::Image, mediapipe::ImageFrame>>::Optional kIn{"IMAGE"};
+  // IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level
+  // graphs for the Python Solution APIs to figure out the type of input stream
+  // without running into ambiguities from IMAGE.
+  // TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image.
+  static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
+  static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
+  static constexpr Output<std::pair<int, int>> kOut{"SIZE"};
 
-    if (cc->Outputs().HasTag("SIZE")) {
-      cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
-    }
+  MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
 
-    return absl::OkStatus();
-  }
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() +
+                     kInGpu(cc).IsConnected(),
+                 1)
+        << "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is "
+           "expected.";
 
-  absl::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) override {
-    int width;
-    int height;
+    std::pair<int, int> size;
 
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
-        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
-      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-      width = image.Width();
-      height = image.Height();
+    if (kIn(cc).IsConnected()) {
+      kIn(cc).Visit(
+          [&size](const mediapipe::Image& value) {
+            size.first = value.width();
+            size.second = value.height();
+          },
+          [&size](const mediapipe::ImageFrame& value) {
+            size.first = value.Width();
+            size.second = value.Height();
+          });
+    }
+    if (kInCpu(cc).IsConnected()) {
+      const auto& image = *kInCpu(cc);
+      size.first = image.Width();
+      size.second = image.Height();
     }
 #if !MEDIAPIPE_DISABLE_GPU
-    if (cc->Inputs().HasTag(kGpuBufferTag) &&
-        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
-      const auto& image =
-          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
-      width = image.width();
-      height = image.height();
+    if (kInGpu(cc).IsConnected()) {
+      const auto& image = *kInGpu(cc);
+      size.first = image.width();
+      size.second = image.height();
     }
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-    cc->Outputs().Tag("SIZE").AddPacket(
-        MakePacket<std::pair<int, int>>(width, height)
-            .At(cc->InputTimestamp()));
+    kOut(cc).Send(size);
 
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(ImagePropertiesCalculator);
 
+MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -546,7 +546,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
   auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
                                                   input.format());
 
-  gpu_helper_.BindFramebuffer(dst);  // GL_TEXTURE0
+  gpu_helper_.BindFramebuffer(dst);
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc

@@ -36,11 +36,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
   Packet input_packet = MakePacket<std::string>(contents);
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));
@@ -79,11 +79,11 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
       reinterpret_cast<const char*>(&encode_buffer[0]), encode_buffer.size())));
 
   CalculatorGraphConfig::Node node_config =
-      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
         calculator: "OpenCvEncodedImageToImageFrameCalculator"
         input_stream: "encoded_image"
         output_stream: "image_frame"
-      )");
+      )pb");
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));

mediapipe/calculators/image/recolor_calculator.cc

@@ -37,6 +37,22 @@ constexpr char kImageFrameTag[] = "IMAGE";
 constexpr char kMaskCpuTag[] = "MASK";
 constexpr char kGpuBufferTag[] = "IMAGE_GPU";
 constexpr char kMaskGpuTag[] = "MASK_GPU";
+
+inline cv::Vec3b Blend(const cv::Vec3b& color1, const cv::Vec3b& color2,
+                       float weight, int invert_mask,
+                       int adjust_with_luminance) {
+  weight = (1 - invert_mask) * weight + invert_mask * (1.0f - weight);
+
+  float luminance =
+      (1 - adjust_with_luminance) * 1.0f +
+      adjust_with_luminance *
+          (color1[0] * 0.299 + color1[1] * 0.587 + color1[2] * 0.114) / 255;
+
+  float mix_value = weight * luminance;
+
+  return color1 * (1.0 - mix_value) + color2 * mix_value;
+}
+
 }  // namespace
 
 namespace mediapipe {
@@ -44,15 +60,14 @@ namespace mediapipe {
 // A calculator to recolor a masked area of an image to a specified color.
 //
 // A mask image is used to specify where to overlay a user defined color.
-// The luminance of the input image is used to adjust the blending weight,
-// to help preserve image textures.
 //
 // Inputs:
 //   One of the following IMAGE tags:
-//   IMAGE: An ImageFrame input image, RGB or RGBA.
+//   IMAGE: An ImageFrame input image in ImageFormat::SRGB.
 //   IMAGE_GPU: A GpuBuffer input image, RGBA.
 //   One of the following MASK tags:
-//   MASK: An ImageFrame input mask, Gray, RGB or RGBA.
+//   MASK: An ImageFrame input mask in ImageFormat::GRAY8, SRGB, SRGBA, or
+//         VEC32F1
 //   MASK_GPU: A GpuBuffer input mask, RGBA.
 // Output:
 //   One of the following IMAGE tags:
@@ -98,10 +113,12 @@ class RecolorCalculator : public CalculatorBase {
   void GlRender();
 
   bool initialized_ = false;
-  std::vector<float> color_;
+  std::vector<uint8> color_;
   mediapipe::RecolorCalculatorOptions::MaskChannel mask_channel_;
 
   bool use_gpu_ = false;
+  bool invert_mask_ = false;
+  bool adjust_with_luminance_ = false;
 #if !MEDIAPIPE_DISABLE_GPU
   mediapipe::GlCalculatorHelper gpu_helper_;
   GLuint program_ = 0;
@@ -209,6 +226,9 @@ absl::Status RecolorCalculator::Close(CalculatorContext* cc) {
 
 absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
   if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
+    cc->Outputs()
+        .Tag(kImageFrameTag)
+        .AddPacket(cc->Inputs().Tag(kImageFrameTag).Value());
     return absl::OkStatus();
   }
   // Get inputs and setup output.
@@ -230,11 +250,15 @@ absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
   }
   cv::Mat mask_full;
   cv::resize(mask_mat, mask_full, input_mat.size());
+  const cv::Vec3b recolor = {color_[0], color_[1], color_[2]};
 
   auto output_img = absl::make_unique<ImageFrame>(
       input_img.Format(), input_mat.cols, input_mat.rows);
   cv::Mat output_mat = mediapipe::formats::MatView(output_img.get());
 
+  const int invert_mask = invert_mask_ ? 1 : 0;
+  const int adjust_with_luminance = adjust_with_luminance_ ? 1 : 0;
+
   // From GPU shader:
   /*
       vec4 weight = texture2D(mask, sample_coordinate);
@@ -246,18 +270,23 @@ absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
 
       fragColor = mix(color1, color2, mix_value);
   */
-  for (int i = 0; i < output_mat.rows; ++i) {
-    for (int j = 0; j < output_mat.cols; ++j) {
-      float weight = mask_full.at<uchar>(i, j) * (1.0 / 255.0);
-      cv::Vec3f color1 = input_mat.at<cv::Vec3b>(i, j);
-      cv::Vec3f color2 = {color_[0], color_[1], color_[2]};
-
-      float luminance =
-          (color1[0] * 0.299 + color1[1] * 0.587 + color1[2] * 0.114) / 255;
-      float mix_value = weight * luminance;
-
-      cv::Vec3b mix_color = color1 * (1.0 - mix_value) + color2 * mix_value;
-      output_mat.at<cv::Vec3b>(i, j) = mix_color;
+  if (mask_img.Format() == ImageFormat::VEC32F1) {
+    for (int i = 0; i < output_mat.rows; ++i) {
+      for (int j = 0; j < output_mat.cols; ++j) {
+        const float weight = mask_full.at<float>(i, j);
+        output_mat.at<cv::Vec3b>(i, j) =
+            Blend(input_mat.at<cv::Vec3b>(i, j), recolor, weight, invert_mask,
+                  adjust_with_luminance);
+      }
+    }
+  } else {
+    for (int i = 0; i < output_mat.rows; ++i) {
+      for (int j = 0; j < output_mat.cols; ++j) {
+        const float weight = mask_full.at<uchar>(i, j) * (1.0 / 255.0);
+        output_mat.at<cv::Vec3b>(i, j) =
+            Blend(input_mat.at<cv::Vec3b>(i, j), recolor, weight, invert_mask,
+                  adjust_with_luminance);
+      }
     }
   }
 
@@ -270,6 +299,9 @@ absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
 
 absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
   if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
+    cc->Outputs()
+        .Tag(kGpuBufferTag)
+        .AddPacket(cc->Inputs().Tag(kGpuBufferTag).Value());
     return absl::OkStatus();
   }
 #if !MEDIAPIPE_DISABLE_GPU
@@ -287,7 +319,7 @@ absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
 
   // Run recolor shader on GPU.
   {
-    gpu_helper_.BindFramebuffer(dst_tex);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(dst_tex);
 
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(img_tex.target(), img_tex.name());
@@ -379,6 +411,9 @@ absl::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) {
   color_.push_back(options.color().g());
   color_.push_back(options.color().b());
 
+  invert_mask_ = options.invert_mask();
+  adjust_with_luminance_ = options.adjust_with_luminance();
+
   return absl::OkStatus();
 }
 
@@ -429,13 +464,20 @@ absl::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
     uniform sampler2D frame;
     uniform sampler2D mask;
     uniform vec3 recolor;
+    uniform float invert_mask;
+    uniform float adjust_with_luminance;
 
     void main() {
       vec4 weight = texture2D(mask, sample_coordinate);
       vec4 color1 = texture2D(frame, sample_coordinate);
       vec4 color2 = vec4(recolor, 1.0);
 
-      float luminance = dot(color1.rgb, vec3(0.299, 0.587, 0.114));
+      weight = mix(weight, 1.0 - weight, invert_mask);
+
+      float luminance = mix(1.0,
+                            dot(color1.rgb, vec3(0.299, 0.587, 0.114)),
+                            adjust_with_luminance);
+
       float mix_value = weight.MASK_COMPONENT * luminance;
 
       fragColor = mix(color1, color2, mix_value);
@@ -452,6 +494,10 @@ absl::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
   glUniform1i(glGetUniformLocation(program_, "mask"), 2);
   glUniform3f(glGetUniformLocation(program_, "recolor"), color_[0] / 255.0,
               color_[1] / 255.0, color_[2] / 255.0);
+  glUniform1f(glGetUniformLocation(program_, "invert_mask"),
+              invert_mask_ ? 1.0f : 0.0f);
+  glUniform1f(glGetUniformLocation(program_, "adjust_with_luminance"),
+              adjust_with_luminance_ ? 1.0f : 0.0f);
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
   return absl::OkStatus();

mediapipe/calculators/image/recolor_calculator.proto

@@ -36,4 +36,11 @@ message RecolorCalculatorOptions {
   // Color to blend into input image where mask is > 0.
   // The blending is based on the input image luminosity.
   optional Color color = 2;
+
+  // Swap the meaning of mask values for foreground/background.
+  optional bool invert_mask = 3 [default = false];
+
+  // Whether to use the luminance of the input image to further adjust the
+  // blending weight, to help preserve image textures.
+  optional bool adjust_with_luminance = 4 [default = true];
 }

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -323,7 +323,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
     const auto& alpha_mask =
         cc->Inputs().Tag(kInputAlphaTagGpu).Get<mediapipe::GpuBuffer>();
     auto alpha_texture = gpu_helper_.CreateSourceTexture(alpha_mask);
-    gpu_helper_.BindFramebuffer(output_texture);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(output_texture);
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(GL_TEXTURE_2D, input_texture.name());
     glActiveTexture(GL_TEXTURE2);
@@ -335,7 +335,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
     glBindTexture(GL_TEXTURE_2D, 0);
     alpha_texture.Release();
   } else {
-    gpu_helper_.BindFramebuffer(output_texture);  // GL_TEXTURE0
+    gpu_helper_.BindFramebuffer(output_texture);
     glActiveTexture(GL_TEXTURE1);
     glBindTexture(GL_TEXTURE_2D, input_texture.name());
     GlRender(cc);  // use value from options

mediapipe/calculators/tensor/BUILD

@@ -490,6 +490,7 @@ cc_library(
         "//mediapipe/framework/port:statusor",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
@@ -526,6 +527,7 @@ mediapipe_proto_library(
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
+        "//mediapipe/gpu:gpu_origin_proto",
     ],
 )
 
@@ -561,13 +563,13 @@ cc_test(
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:commandlineflags",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],
@@ -583,6 +585,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_utils",
         "//mediapipe/framework/formats:image",
@@ -750,3 +753,76 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
     ],
 )
+
+# Copied from /mediapipe/calculators/tflite/BUILD
+selects.config_setting_group(
+    name = "gpu_inference_disabled",
+    match_any = [
+        "//mediapipe/gpu:disable_gpu",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "tensors_to_segmentation_calculator_proto",
+    srcs = ["tensors_to_segmentation_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/gpu:gpu_origin_proto",
+    ],
+)
+
+cc_library(
+    name = "tensors_to_segmentation_calculator",
+    srcs = ["tensors_to_segmentation_calculator.cc"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tensors_to_segmentation_calculator_cc_proto",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_opencv",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:port",
+        "//mediapipe/util:resource_util",
+        "@org_tensorflow//tensorflow/lite:framework",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
+        "//mediapipe/framework/port:statusor",
+    ] + selects.with_or({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gl_calculator_helper",
+            "//mediapipe/gpu:gl_simple_shaders",
+            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:shader_util",
+        ],
+    }) + selects.with_or({
+        ":gpu_inference_disabled": [],
+        "//mediapipe:ios": [
+            "//mediapipe/gpu:MPPMetalUtil",
+            "//mediapipe/gpu:MPPMetalHelper",
+        ],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util",
+        ],
+    }),
+    alwayslink = 1,
+)

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -31,6 +31,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
@@ -236,7 +237,7 @@ class ImageToTensorCalculator : public Node {
   }
 
  private:
-  bool DoesInputStartAtBottom() {
+  bool DoesGpuInputStartAtBottom() {
     return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
   }
 
@@ -290,11 +291,11 @@ class ImageToTensorCalculator : public Node {
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
         ASSIGN_OR_RETURN(gpu_converter_,
                          CreateImageToGlBufferTensorConverter(
-                             cc, DoesInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #else
         ASSIGN_OR_RETURN(gpu_converter_,
                          CreateImageToGlTextureTensorConverter(
-                             cc, DoesInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #endif  // MEDIAPIPE_METAL_ENABLED
 #endif  // !MEDIAPIPE_DISABLE_GPU
       }

mediapipe/calculators/tensor/image_to_tensor_calculator.proto

@@ -17,20 +17,7 @@ syntax = "proto2";
 package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
-
-message GpuOrigin {
-  enum Mode {
-    DEFAULT = 0;
-
-    // OpenGL: bottom-left origin
-    // Metal : top-left origin
-    CONVENTIONAL = 1;
-
-    // OpenGL: top-left origin
-    // Metal : top-left origin
-    TOP_LEFT = 2;
-  }
-}
+import "mediapipe/gpu/gpu_origin.proto";
 
 message ImageToTensorCalculatorOptions {
   extend mediapipe.CalculatorOptions {

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -15,6 +15,7 @@
 #include <cmath>
 #include <vector>
 
+#include "absl/flags/flag.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
@@ -28,7 +29,6 @@
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
-#include "mediapipe/framework/port/commandlineflags.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter {
       return absl::OkStatus();
     }));
 
-    return tensor;
+    return std::move(tensor);
   }
 
   ~GlProcessor() override {

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
           tflite::gpu::HW(output_dims.height, output_dims.width),
           command_buffer, buffer_view.buffer()));
       [command_buffer commit];
-      return tensor;
+      return std::move(tensor);
     }
   }
 

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc

@@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
         GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                     range_min, range_max));
     transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
-    return tensor;
+    return std::move(tensor);
   }
 
  private:

mediapipe/calculators/tensor/inference_calculator.cc

@@ -41,7 +41,6 @@ class InferenceCalculatorSelectorImpl
         (options.has_delegate() && options.delegate().has_gpu());
     if (should_use_gpu) {
       impls.emplace_back("Metal");
-      impls.emplace_back("MlDrift");
       impls.emplace_back("Gl");
     }
     impls.emplace_back("Cpu");

mediapipe/calculators/tensor/inference_calculator.h

@@ -118,10 +118,6 @@ struct InferenceCalculatorGl : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
 };
 
-struct InferenceCalculatorMlDrift : public InferenceCalculator {
-  static constexpr char kCalculatorName[] = "InferenceCalculatorMlDrift";
-};
-
 struct InferenceCalculatorMetal : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
 };

mediapipe/calculators/tensor/inference_calculator.proto

@@ -51,12 +51,12 @@ message InferenceCalculatorOptions {
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Choose any of available APIs to force running inference using it.
-      enum API {
+      enum Api {
         ANY = 0;
         OPENGL = 1;
         OPENCL = 2;
       }
-      optional API api = 4 [default = ANY];
+      optional Api api = 4 [default = ANY];
 
       // This option is valid for TFLite GPU delegate API2 only,
       // Set to true to use 16-bit float precision. If max precision is needed,