Project import generated by Copybara.

GitOrigin-RevId: 6e4aff1cc351be3ae4537b677f36d139ee50ce09

.bazelrc

@@ -5,9 +5,12 @@ common --experimental_repo_remote_exec
 
 # Basic build settings
 build --jobs 128
-build --define='absl=1'
+build --define='absl=1'  # for gtest
 build --enable_platform_specific_config
 
+# Enable stack traces
+test --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1"
+
 # Linux
 build:linux --cxxopt=-std=c++17
 build:linux --host_cxxopt=-std=c++17

.bazelversion

@@ -0,0 +1 @@
+3.7.2

Dockerfile

@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.4.1
+ARG BAZEL_VERSION=3.7.2
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

MANIFEST.in

@@ -8,3 +8,9 @@ include README.md
 include requirements.txt
 
 recursive-include mediapipe/modules *.tflite *.txt *.binarypb
+exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
+exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite

README.md

@@ -34,7 +34,7 @@ Hair Segmentation
 
 []()                                                                                      | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
 :---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
-[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
+[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             | ✅
 [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)                             | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Iris](https://google.github.io/mediapipe/solutions/iris)                                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
@@ -102,6 +102,8 @@ run code search using
 
 ## Publications
 
+*   [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
+    in Google AI Blog
 *   [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
     in Google AI Blog
 *   [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)

WORKSPACE

@@ -2,16 +2,19 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
-    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+    urls = [
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+    ],
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
 )
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "3.4.0")
-
+versions.check(minimum_bazel_version = "3.7.2")
 
 # ABSL cpp library lts_2020_09_23
 http_archive(
@@ -38,8 +41,8 @@ http_archive(
 
 http_archive(
    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-master",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip",
+   strip_prefix = "rules_foreign_cc-0.1.0",
+   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -304,8 +307,8 @@ http_archive(
 
 # Maven dependencies.
 
-RULES_JVM_EXTERNAL_TAG = "3.2"
-RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+RULES_JVM_EXTERNAL_TAG = "4.0"
+RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
 
 http_archive(
     name = "rules_jvm_external",
@@ -318,7 +321,6 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 
 # Important: there can only be one maven_install rule. Add new maven deps here.
 maven_install(
-    name = "maven",
     artifacts = [
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.2.0",
@@ -343,10 +345,10 @@ maven_install(
         "org.hamcrest:hamcrest-library:1.3",
     ],
     repositories = [
-        "https://jcenter.bintray.com",
         "https://maven.google.com",
         "https://dl.google.com/dl/android/maven2",
         "https://repo1.maven.org/maven2",
+        "https://jcenter.bintray.com",
     ],
     fetch_sources = True,
     version_conflict_policy = "pinned",
@@ -363,10 +365,10 @@ http_archive(
     ],
 )
 
-#Tensorflow repo should always go after the other external dependencies.
-# 2020-12-07
-_TENSORFLOW_GIT_COMMIT = "f556709f4df005ad57fd24d5eaa0d9380128d3ba"
-_TENSORFLOW_SHA256= "9e157d4723921b48a974f645f70d07c8fd3c363569a0ac6ee85fec114d6459ea"
+# Tensorflow repo should always go after the other external dependencies.
+# 2021-03-25
+_TENSORFLOW_GIT_COMMIT = "c67f68021824410ebe9f18513b8856ac1c6d4887"
+_TENSORFLOW_SHA256= "fd07d0b39422dc435e268c5e53b2646a8b4b1e3151b87837b43f86068faae87f"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -383,5 +385,7 @@ http_archive(
     sha256 = _TENSORFLOW_SHA256,
 )
 
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace(tf_repo_name = "org_tensorflow")
+load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
+tf_workspace3()
+load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
+tf_workspace2()

build_desktop_examples.sh

@@ -94,7 +94,8 @@ for app in ${apps}; do
       else
         graph_name="${target_name}/${target_name}"
       fi
-      if [[ ${target_name} == "iris_tracking" ||
+      if [[ ${target_name} == "holistic_tracking" ||
+            ${target_name} == "iris_tracking" ||
             ${target_name} == "pose_tracking" ||
             ${target_name} == "upper_body_pose_tracking" ]]; then
         graph_suffix="cpu"

docs/framework_concepts/calculators.md

@@ -67,26 +67,26 @@ class CalculatorBase {
 
   // The subclasses of CalculatorBase must implement GetContract.
   // ...
-  static ::MediaPipe::Status GetContract(CalculatorContract* cc);
+  static absl::Status GetContract(CalculatorContract* cc);
 
   // Open is called before any Process() calls, on a freshly constructed
   // calculator.  Subclasses may override this method to perform necessary
   // setup, and possibly output Packets and/or set output streams' headers.
   // ...
-  virtual ::MediaPipe::Status Open(CalculatorContext* cc) {
-    return ::MediaPipe::OkStatus();
+  virtual absl::Status Open(CalculatorContext* cc) {
+    return absl::OkStatus();
   }
 
   // Processes the incoming inputs. May call the methods on cc to access
   // inputs and produce outputs.
   // ...
-  virtual ::MediaPipe::Status Process(CalculatorContext* cc) = 0;
+  virtual absl::Status Process(CalculatorContext* cc) = 0;
 
   // Is called if Open() was called and succeeded.  Is called either
   // immediately after processing is complete or after a graph run has ended
   // (if an error occurred in the graph).  ...
-  virtual ::MediaPipe::Status Close(CalculatorContext* cc) {
-    return ::MediaPipe::OkStatus();
+  virtual absl::Status Close(CalculatorContext* cc) {
+    return absl::OkStatus();
   }
 
   ...
@@ -199,7 +199,7 @@ name and index number. In the function below input are output are identified:
 // c++ Code snippet describing the SomeAudioVideoCalculator GetContract() method
 class SomeAudioVideoCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).SetAny();
     // SetAny() is used to specify that whatever the type of the
     // stream is, it's acceptable.  This does not mean that any
@@ -209,13 +209,13 @@ class SomeAudioVideoCalculator : public CalculatorBase {
     cc->Outputs().Tag("VIDEO").Set<ImageFrame>();
     cc->Outputs().Get("AUDIO", 0).Set<Matrix>();
     cc->Outputs().Get("AUDIO", 1).Set<Matrix>();
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 ```
 
 ## Processing
 
-`Process()` called on a non-source node must return `::mediapipe::OkStatus()` to
+`Process()` called on a non-source node must return `absl::OkStatus()` to
 indicate that all went well, or any other status code to signal an error
 
 If a non-source calculator returns `tool::StatusStop()`, then this signals the
@@ -224,12 +224,12 @@ input streams will be closed (and remaining Packets will propagate through the
 graph).
 
 A source node in a graph will continue to have `Process()` called on it as long
-as it returns `::mediapipe::OkStatus(`). To indicate that there is no more data
-to be generated return `tool::StatusStop()`. Any other status indicates an error
-has occurred.
+as it returns `absl::OkStatus(`). To indicate that there is no more data to be
+generated return `tool::StatusStop()`. Any other status indicates an error has
+occurred.
 
-`Close()` returns `::mediapipe::OkStatus()` to indicate success. Any other
-status indicates a failure.
+`Close()` returns `absl::OkStatus()` to indicate success. Any other status
+indicates a failure.
 
 Here is the basic `Process()` function. It uses the `Input()` method (which can
 be used only if the calculator has a single input) to request its input data. It
@@ -238,13 +238,13 @@ and does the calculations. When done it releases the pointer when adding it to
 the output stream.
 
 ```c++
-::util::Status MyCalculator::Process() {
+absl::Status MyCalculator::Process() {
   const Matrix& input = Input()->Get<Matrix>();
   std::unique_ptr<Matrix> output(new Matrix(input.rows(), input.cols()));
   // do your magic here....
   //    output->row(n) =  ...
   Output()->Add(output.release(), InputTimestamp());
-  return ::mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 ```
 
@@ -312,7 +312,7 @@ namespace mediapipe {
 //
 class PacketClonerCalculator : public CalculatorBase {
  public:
-  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     const int tick_signal_index = cc->Inputs().NumEntries() - 1;
     // cc->Inputs().NumEntries() returns the number of input streams
     // for the PacketClonerCalculator
@@ -322,10 +322,10 @@ class PacketClonerCalculator : public CalculatorBase {
       cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
     }
     cc->Inputs().Index(tick_signal_index).SetAny();
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  ::mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     tick_signal_index_ = cc->Inputs().NumEntries() - 1;
     current_.resize(tick_signal_index_);
     // Pass along the header for each stream if present.
@@ -336,10 +336,10 @@ class PacketClonerCalculator : public CalculatorBase {
         // the header for the input stream of index i
       }
     }
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  ::mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     // Store input signals.
     for (int i = 0; i < tick_signal_index_; ++i) {
       if (!cc->Inputs().Index(i).Value().IsEmpty()) {
@@ -364,7 +364,7 @@ class PacketClonerCalculator : public CalculatorBase {
         }
       }
     }
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

docs/framework_concepts/gpu.md

@@ -66,10 +66,10 @@ calculator derived from base class GlSimpleCalculator. The GPU calculator
 // See GlSimpleCalculator for inputs, outputs and input side packets.
 class LuminanceCalculator : public GlSimpleCalculator {
  public:
-  ::mediapipe::Status GlSetup() override;
-  ::mediapipe::Status GlRender(const GlTexture& src,
-                               const GlTexture& dst) override;
-  ::mediapipe::Status GlTeardown() override;
+  absl::Status GlSetup() override;
+  absl::Status GlRender(const GlTexture& src,
+                        const GlTexture& dst) override;
+  absl::Status GlTeardown() override;
 
  private:
   GLuint program_ = 0;
@@ -77,8 +77,8 @@ class LuminanceCalculator : public GlSimpleCalculator {
 };
 REGISTER_CALCULATOR(LuminanceCalculator);
 
-::mediapipe::Status LuminanceCalculator::GlRender(const GlTexture& src,
-                                                  const GlTexture& dst) {
+absl::Status LuminanceCalculator::GlRender(const GlTexture& src,
+                                           const GlTexture& dst) {
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -128,7 +128,7 @@ REGISTER_CALCULATOR(LuminanceCalculator);
   glDeleteVertexArrays(1, &vao);
   glDeleteBuffers(2, vbo);
 
-  return ::mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 ```
 

docs/framework_concepts/graphs.md

@@ -219,23 +219,23 @@ packet timestamps 0, 1, 2, 3, ...
 ```c++
 class UnitDelayCalculator : public Calculator {
  public:
-  static ::util::Status FillExpectations(
+  static absl::Status FillExpectations(
       const CalculatorOptions& extendable_options, PacketTypeSet* inputs,
       PacketTypeSet* outputs, PacketTypeSet* input_side_packets) {
     inputs->Index(0)->Set<int>("An integer.");
     outputs->Index(0)->Set<int>("The input delayed by one time unit.");
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  ::util::Status Open() final {
+  absl::Status Open() final {
     Output()->Add(new int(0), Timestamp(0));
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  ::util::Status Process() final {
+  absl::Status Process() final {
     const Packet& packet = Input()->Value();
     Output()->AddPacket(packet.At(packet.Timestamp().NextAllowedInStream()));
-    return ::mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 ```

docs/framework_concepts/packets.md

@@ -12,19 +12,30 @@ nav_order: 3
 {:toc}
 ---
 
-Each calculator is a node of of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options
+Calculators communicate by sending and receiving packets. Typically a single
+packet is sent along each input stream at each input timestamp. A packet can
+contain any kind of data, such as a single frame of video or a single integer
+detection count.
 
 ## Creating a packet
 
-Packets are generally created with `MediaPipe::Adopt()` (from packet.h).
+Packets are generally created with `mediapipe::MakePacket<T>()` or
+`mediapipe::Adopt()` (from packet.h).
 
 ```c++
-// Create some data.
-auto data = absl::make_unique<MyDataClass>("constructor_argument");
-// Create a packet to own the data.
-Packet p = Adopt(data.release());
+// Create a packet containing some new data.
+Packet p = MakePacket<MyDataClass>("constructor_argument");
 // Make a new packet with the same data and a different timestamp.
 Packet p2 = p.At(Timestamp::PostStream());
 ```
 
+or:
+
+```c++
+// Create some new data.
+auto data = absl::make_unique<MyDataClass>("constructor_argument");
+// Create a packet to own the data.
+Packet p = Adopt(data.release()).At(Timestamp::PostStream());
+```
+
 Data within a packet is accessed with `Packet::Get<T>()`

docs/getting_started/android.md

@@ -28,7 +28,7 @@ Gradle.
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
 *   Setup Android SDK release 28.0.3 and above.
-*   Setup Android NDK r18b and above.
+*   Setup Android NDK version between 18 and 21.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
 below for Android Studio setup). However, if you prefer using MediaPipe without

docs/getting_started/android_archive_library.md

@@ -45,7 +45,8 @@ each project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+        --fat_apk_cpu=arm64-v8a,armeabi-v7a --strip=ALWAYS \
         //path/to/the/aar/build/file:aar_name
     ```
 
@@ -86,16 +87,14 @@ each project.
     Build the MediaPipe binary graph and copy the assets into
     app/src/main/assets, e.g., for the face detection graph, you need to build
     and copy
-    [the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41),
-    [the tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
+    [the binary graph](https://github.com/google/mediapipe/blob/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD#L41)
     and
-    [the label map](https://github.com/google/mediapipe/blob/master/mediapipe/models/face_detection_front_labelmap.txt).
+    [the face detection tflite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite).
 
     ```bash
     bazel build -c opt mediapipe/mediapipe/graphs/face_detection:mobile_gpu_binary_graph
     cp bazel-bin/mediapipe/graphs/face_detection/mobile_gpu.binarypb /path/to/your/app/src/main/assets/
-    cp mediapipe/models/face_detection_front.tflite /path/to/your/app/src/main/assets/
-    cp mediapipe/models/face_detection_front_labelmap.txt /path/to/your/app/src/main/assets/
+    cp mediapipe/modules/face_detection/face_detection_front.tflite /path/to/your/app/src/main/assets/
     ```
 
     ![Screenshot](../images/mobile/assets_location.png)

docs/getting_started/hello_world_android.md

@@ -59,7 +59,7 @@ node: {
   output_stream: "luma_video"
 }
 
-# Applies the Sobel filter to luminance images sotred in RGB format.
+# Applies the Sobel filter to luminance images stored in RGB format.
 node: {
   calculator: "SobelEdgesCalculator"
   input_stream: "luma_video"

docs/getting_started/hello_world_cpp.md

@@ -44,7 +44,7 @@ nav_order: 1
     `PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
 
     ```C++
-    ::mediapipe::Status PrintHelloWorld() {
+    absl::Status PrintHelloWorld() {
       // Configures a simple graph, which concatenates 2 PassThroughCalculators.
       CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
         input_stream: "in"

docs/getting_started/hello_world_ios.md

@@ -492,6 +492,9 @@ in our app:
     if (![self.mediapipeGraph startWithError:&error]) {
       NSLog(@"Failed to start graph: %@", error);
     }
+    else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) {
+      NSLog(@"Failed to complete graph initial run: %@", error);
+    }
 
     dispatch_async(_videoQueue, ^{
       [_cameraSource start];
@@ -500,8 +503,9 @@ in our app:
 }];
 ```
 
-Note: It is important to start the graph before starting the camera, so that the
-graph is ready to process frames as soon as the camera starts sending them.
+Note: It is important to start the graph before starting the camera and wait
+until completion, so that the graph is ready to process frames as soon as the
+camera starts sending them.
 
 Earlier, when we received frames from the camera in the `processVideoFrame`
 function, we displayed them in the `_liveView` using the `_renderer`. Now, we

docs/getting_started/install.md

@@ -25,25 +25,11 @@ install --user six`.
 
 ## Installing on Debian and Ubuntu
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install Bazel 3.4 or higher.
-
-    For Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu only, Bazel needs
-    to be built from source:
-
-    ```bash
-    # For Bazel 3.4.0
-    mkdir $HOME/bazel-3.4.0
-    cd $HOME/bazel-3.4.0
-    wget https://github.com/bazelbuild/bazel/releases/download/3.4.0/bazel-3.4.0-dist.zip
-    sudo apt-get install build-essential openjdk-8-jdk python zip unzip
-    unzip bazel-3.4.0-dist.zip
-    env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
-    sudo cp output/bazel /usr/local/bin/
-    ```
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -207,11 +193,11 @@ build issues.
 
 **Disclaimer**: Running MediaPipe on CentOS is experimental.
 
-1.  Install Bazel.
+1.  Install Bazelisk.
 
     Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel 3.4 or higher.
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 2.  Checkout MediaPipe repository.
 
@@ -336,18 +322,11 @@ build issues.
     *   Install [Xcode](https://developer.apple.com/xcode/) and its Command Line
         Tools by `xcode-select --install`.
 
-2.  Install Bazel.
+2.  Install Bazelisk.
 
-    Option 1. Use package manager tool to install Bazel
-
-    ```bash
-    $ brew install bazel
-    # Run 'bazel version' to check version of bazel
-    ```
-
-    Option 2. Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install Bazel 3.4 or higher.
+    Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 3.  Checkout MediaPipe repository.
 
@@ -360,7 +339,7 @@ build issues.
 4.  Install OpenCV and FFmpeg.
 
     Option 1. Use HomeBrew package manager tool to install the pre-compiled
-    OpenCV 3.4.5 libraries. FFmpeg will be installed via OpenCV.
+    OpenCV 3 libraries. FFmpeg will be installed via OpenCV.
 
     ```bash
     $ brew install opencv@3
@@ -491,29 +470,36 @@ next section.
 
 4.  Install Visual C++ Build Tools 2019 and WinSDK
 
-    Go to https://visualstudio.microsoft.com/visual-cpp-build-tools, download
-    build tools, and install Microsoft Visual C++ 2019 Redistributable and
-    Microsoft Build Tools 2019.
+    Go to
+    [the VisualStudio website](ttps://visualstudio.microsoft.com/visual-cpp-build-tools),
+    download build tools, and install Microsoft Visual C++ 2019 Redistributable
+    and Microsoft Build Tools 2019.
 
     Download the WinSDK from
-    https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/ and
-    install.
+    [the official MicroSoft website](https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk/)
+    and install.
 
-5.  Install Bazel and add the location of the Bazel executable to the `%PATH%`
-    environment variable.
+5.  Install Bazel or Bazelisk and add the location of the Bazel executable to
+    the `%PATH%` environment variable.
 
-    Follow the official
-    [Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.4 or higher.
+    Option 1. Follow
+    [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
+    to install Bazel 3.7.2 or higher.
 
-6.  Set Bazel variables.
+    Option 2. Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
+
+6.  Set Bazel variables. Learn more details about
+    ["Build on Windows"](https://docs.bazel.build/versions/master/windows.html#build-c-with-msvc)
+    in the Bazel official documentation.
 
     ```
-    # Find the exact paths and version numbers from your local version.
+    # Please find the exact paths and version numbers from your local version.
     C:\> set BAZEL_VS=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools
     C:\> set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC
-    C:\> set BAZEL_VC_FULL_VERSION=14.25.28610
-    C:\> set BAZEL_WINSDK_FULL_VERSION=10.1.18362.1
+    C:\> set BAZEL_VC_FULL_VERSION=<Your local VC version>
+    C:\> set BAZEL_WINSDK_FULL_VERSION=<Your local WinSDK version>
     ```
 
 7.  Checkout MediaPipe repository.
@@ -600,19 +586,11 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel.
+5.  Install Bazelisk.
 
-    ```bash
-    username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/3.4.0/release/bazel-3.4.0-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/3.4.0 && \
-    chmod 755 bazel-3.4.0-installer-linux-x86_64.sh && \
-    sudo ./bazel-3.4.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.0 && \
-    source /usr/local/bazel/3.4.0/lib/bazel/bin/bazel-complete.bash
-
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.0/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/3.4.0/lib/bazel/bin/bazel'
-    ```
+    Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
+    to install Bazelisk.
 
 6.  Checkout MediaPipe repository.
 

docs/getting_started/javascript.md

@@ -19,9 +19,10 @@ MediaPipe currently offers the following solutions:
 Solution          | NPM Package                   | Example
 ----------------- | ----------------------------- | -------
 [Face Mesh][F-pg] | [@mediapipe/face_mesh][F-npm] | [mediapipe.dev/demo/face_mesh][F-demo]
+[Face Detection][Fd-pg] | [@mediapipe/face_detection][Fd-npm] | [mediapipe.dev/demo/face_detection][Fd-demo]
 [Hands][H-pg]     | [@mediapipe/hands][H-npm]     | [mediapipe.dev/demo/hands][H-demo]
-[Pose][P-pg]      | [@mediapipe/pose][P-npm]      | [mediapipe.dev/demo/pose][P-demo]
 [Holistic][Ho-pg] | [@mediapipe/holistic][Ho-npm] | [mediapipe.dev/demo/holistic][Ho-demo]
+[Pose][P-pg]      | [@mediapipe/pose][P-npm]      | [mediapipe.dev/demo/pose][P-demo]
 
 Click on a solution link above for more information, including API and code
 snippets.
@@ -63,10 +64,12 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 
 [Ho-pg]: ../solutions/holistic#javascript-solution-api
 [F-pg]: ../solutions/face_mesh#javascript-solution-api
+[Fd-pg]: ../solutions/face_detection#javascript-solution-api
 [H-pg]: ../solutions/hands#javascript-solution-api
 [P-pg]: ../solutions/pose#javascript-solution-api
 [Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic
 [F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh
+[Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
 [H-npm]: https://www.npmjs.com/package/@mediapipe/hands
 [P-npm]: https://www.npmjs.com/package/@mediapipe/pose
 [draw-npm]: https://www.npmjs.com/package/@mediapipe/pose
@@ -74,14 +77,17 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 [ctrl-npm]: https://www.npmjs.com/package/@mediapipe/pose
 [Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
 [F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
+[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
 [H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands
 [P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose
 [Ho-pen]: https://code.mediapipe.dev/codepen/holistic
 [F-pen]: https://code.mediapipe.dev/codepen/face_mesh
+[Fd-pen]: https://code.mediapipe.dev/codepen/face_detection
 [H-pen]: https://code.mediapipe.dev/codepen/hands
 [P-pen]: https://code.mediapipe.dev/codepen/pose
 [Ho-demo]: https://mediapipe.dev/demo/holistic
 [F-demo]: https://mediapipe.dev/demo/face_mesh
+[Fd-demo]: https://mediapipe.dev/demo/face_detection
 [H-demo]: https://mediapipe.dev/demo/hands
 [P-demo]: https://mediapipe.dev/demo/pose
 [npm]: https://www.npmjs.com/package/@mediapipe

docs/getting_started/python.md

@@ -45,17 +45,23 @@ Tip: Use command `deactivate` to later exit the Python virtual environment.
 To learn more about configuration options and usage examples, please find
 details in each solution via the links below:
 
+*   [MediaPipe Face Detection](../solutions/face_detection#python-solution-api)
 *   [MediaPipe Face Mesh](../solutions/face_mesh#python-solution-api)
 *   [MediaPipe Hands](../solutions/hands#python-solution-api)
-*   [MediaPipe Pose](../solutions/pose#python-solution-api)
 *   [MediaPipe Holistic](../solutions/holistic#python-solution-api)
+*   [MediaPipe Objectron](../solutions/objectron#python-solution-api)
+*   [MediaPipe Pose](../solutions/pose#python-solution-api)
 
 ## MediaPipe on Google Colab
 
+*   [MediaPipe Face Detection Colab](https://mediapipe.page.link/face_detection_py_colab)
 *   [MediaPipe Face Mesh Colab](https://mediapipe.page.link/face_mesh_py_colab)
 *   [MediaPipe Hands Colab](https://mediapipe.page.link/hands_py_colab)
-*   [MediaPipe Pose Colab](https://mediapipe.page.link/pose_py_colab)
 *   [MediaPipe Holistic Colab](https://mediapipe.page.link/holistic_py_colab)
+*   [MediaPipe Objectron Colab](https://mediapipe.page.link/objectron_py_colab)
+*   [MediaPipe Pose Colab](https://mediapipe.page.link/pose_py_colab)
+*   [MediaPipe Pose Classification Colab (Basic)](https://mediapipe.page.link/pose_classification_basic)
+*   [MediaPipe Pose Classification Colab (Extended)](https://mediapipe.page.link/pose_classification_extended)
 
 ## MediaPipe Python Framework
 
@@ -70,6 +76,11 @@ Python package from source. Otherwise, we strongly encourage our users to simply
 run `pip install mediapipe` to use the ready-to-use solutions, more convenient
 and much faster.
 
+MediaPipe PyPI currently doesn't provide aarch64 Python wheel
+files. For building and using MediaPipe Python on aarch64 Linux systems such as
+Nvidia Jetson and Raspberry Pi, please read
+[here](https://github.com/jiuqiant/mediapipe-python-aarch64).
+
 1.  Make sure that Bazel and OpenCV are correctly installed and configured for
     MediaPipe. Please see [Installation](./install.md) for how to setup Bazel
     and OpenCV for MediaPipe on Linux and macOS.
@@ -82,12 +93,18 @@ and much faster.
     $ sudo apt install python3-dev
     $ sudo apt install python3-venv
     $ sudo apt install -y protobuf-compiler
+
+    # If you need to build opencv from source.
+    $ sudo apt install cmake
     ```
 
     macOS:
 
     ```bash
     $ brew install protobuf
+
+    # If you need to build opencv from source.
+    $ brew install cmake
     ```
 
     Windows:
@@ -118,3 +135,10 @@ and much faster.
     (mp_env)mediapipe$ python3 setup.py gen_protos
     (mp_env)mediapipe$ python3 setup.py install --link-opencv
     ```
+
+    or
+
+    ```bash
+    (mp_env)mediapipe$ python3 setup.py gen_protos
+    (mp_env)mediapipe$ python3 setup.py bdist_wheel
+    ```

docs/index.md

@@ -34,7 +34,7 @@ Hair Segmentation
 
 []()                                                                                      | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
 :---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
-[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
+[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             | ✅
 [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)                             | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Iris](https://google.github.io/mediapipe/solutions/iris)                                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
@@ -44,7 +44,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                        | ✅                                                         | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
@@ -102,6 +102,8 @@ run code search using
 
 ## Publications
 
+*   [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
+    in Google AI Blog
 *   [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
     in Google AI Blog
 *   [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)

docs/solutions/face_detection.md

@@ -39,6 +39,169 @@ section.
 
 ![face_detection_android_gpu.gif](../images/mobile/face_detection_android_gpu.gif)
 
+## Solution APIs
+
+### Configuration Options
+
+Naming style and availability may differ slightly across platforms/languages.
+
+#### min_detection_confidence
+
+Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
+detection to be considered successful. Default to `0.5`.
+
+### Output
+
+Naming style may differ slightly across platforms/languages.
+
+#### detections
+
+Collection of detected faces, where each face is represented as a detection
+proto message that contains a bounding box and 6 key points (right eye, left
+eye, nose tip, mouth center, right ear tragion, and left ear tragion). The
+bounding box is composed of `xmin` and `width` (both normalized to `[0.0, 1.0]`
+by the image width) and `ymin` and `height` (both normalized to `[0.0, 1.0]` by
+the image height). Each key point is composed of `x` and `y`, which are
+normalized to `[0.0, 1.0]` by the image width and height respectively.
+
+### Python Solution API
+
+Please first follow general [instructions](../getting_started/python.md) to
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
+
+Supported configuration options:
+
+*   [min_detection_confidence](#min_detection_confidence)
+
+```python
+import cv2
+import mediapipe as mp
+mp_face_detction = mp.solutions.face_detection
+
+# For static images:
+with mp_face_detection.FaceDetection(
+    min_detection_confidence=0.5) as face_detection:
+  for idx, file in enumerate(file_list):
+    image = cv2.imread(file)
+    # Convert the BGR image to RGB and process it with MediaPipe Face Detection.
+    results = face_detection.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+
+    # Draw face detections of each face.
+    if not results.detections:
+      continue
+    annotated_image = image.copy()
+    for detection in results.detections:
+      print('Nose tip:')
+      print(mp_face_detection.get_key_point(
+          detection, mp_face_detection.FaceKeyPoint.NOSE_TIP))
+      mp_drawing.draw_detection(annotated_image, detection)
+    cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
+
+# For webcam input:
+cap = cv2.VideoCapture(0)
+with mp_face_detection.FaceDetection(
+    min_detection_confidence=0.5) as face_detection:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
+
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = face_detection.process(image)
+
+    # Draw the face detection annotations on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    if results.detections:
+      for detection in results.detections:
+        mp_drawing.draw_detection(image, detection)
+    cv2.imshow('MediaPipe Face Detection', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
+cap.release()
+```
+
+### JavaScript Solution API
+
+Please first see general [introduction](../getting_started/javascript.md) on
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
+
+Supported configuration options:
+
+*   [minDetectionConfidence](#min_detection_confidence)
+
+```html
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/face_detection/face_detection.js" crossorigin="anonymous"></script>
+</head>
+
+<body>
+  <div class="container">
+    <video class="input_video"></video>
+    <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+  </div>
+</body>
+</html>
+```
+
+```javascript
+<script type="module">
+const videoElement = document.getElementsByClassName('input_video')[0];
+const canvasElement = document.getElementsByClassName('output_canvas')[0];
+const canvasCtx = canvasElement.getContext('2d');
+
+function onResults(results) {
+  // Draw the overlays.
+  canvasCtx.save();
+  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(
+      results.image, 0, 0, canvasElement.width, canvasElement.height);
+  if (results.detections.length > 0) {
+    drawingUtils.drawRectangle(
+        canvasCtx, results.detections[0].boundingBox,
+        {color: 'blue', lineWidth: 4, fillColor: '#00000000'});
+    drawingUtils.drawLandmarks(canvasCtx, results.detections[0].landmarks, {
+      color: 'red',
+      radius: 5,
+    });
+  }
+  canvasCtx.restore();
+}
+
+const faceDetection = new FaceDetection({locateFile: (file) => {
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
+}});
+faceDetection.setOptions({
+  minDetectionConfidence: 0.5
+});
+faceDetection.onResults(onResults);
+
+const camera = new Camera(videoElement, {
+  onFrame: async () => {
+    await faceDetection.send({image: videoElement});
+  },
+  width: 1280,
+  height: 720
+});
+camera.start();
+</script>
+```
+
 ## Example Apps
 
 Please first see general instructions for
@@ -108,3 +271,5 @@ to cross-compile and run MediaPipe examples on the
     ([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
     ([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
 *   [Models and model cards](./models.md#face_detection)
+*   [Web demo](https://code.mediapipe.dev/codepen/face_detection)
+*   [Python Colab](https://mediapipe.page.link/face_detection_py_colab)

docs/solutions/face_mesh.md

@@ -185,8 +185,8 @@ following steps are executed in the given order:
 The geometry pipeline is implemented as a MediaPipe
 [calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
 For your convenience, the face geometry pipeline calculator is bundled together
-with the face landmark module into a unified MediaPipe
-[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
+with corresponding metadata into a unified MediaPipe
+[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt).
 The face geometry format is defined as a Protocol Buffer
 [message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
 
@@ -264,8 +264,8 @@ magnitude of `z` uses roughly the same scale as `x`.
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
-install MediaPipe Python package, then learn more in the companion [Colab] and
-the following usage example.
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
 
 Supported configuration options:
 
@@ -281,74 +281,73 @@ mp_drawing = mp.solutions.drawing_utils
 mp_face_mesh = mp.solutions.face_mesh
 
 # For static images:
-face_mesh = mp_face_mesh.FaceMesh(
+drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+with mp_face_mesh.FaceMesh(
     static_image_mode=True,
     max_num_faces=1,
-    min_detection_confidence=0.5)
-drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
-for idx, file in enumerate(file_list):
-  image = cv2.imread(file)
-  # Convert the BGR image to RGB before processing.
-  results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+    min_detection_confidence=0.5) as face_mesh:
+  for idx, file in enumerate(file_list):
+    image = cv2.imread(file)
+    # Convert the BGR image to RGB before processing.
+    results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
 
-  # Print and draw face mesh landmarks on the image.
-  if not results.multi_face_landmarks:
-    continue
-  annotated_image = image.copy()
-  for face_landmarks in results.multi_face_landmarks:
-    print('face_landmarks:', face_landmarks)
-    mp_drawing.draw_landmarks(
-        image=annotated_image,
-        landmark_list=face_landmarks,
-        connections=mp_face_mesh.FACE_CONNECTIONS,
-        landmark_drawing_spec=drawing_spec,
-        connection_drawing_spec=drawing_spec)
-  cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
-face_mesh.close()
-
-# For webcam input:
-face_mesh = mp_face_mesh.FaceMesh(
-    min_detection_confidence=0.5, min_tracking_confidence=0.5)
-drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
-cap = cv2.VideoCapture(0)
-while cap.isOpened():
-  success, image = cap.read()
-  if not success:
-    print("Ignoring empty camera frame.")
-    # If loading a video, use 'break' instead of 'continue'.
-    continue
-
-  # Flip the image horizontally for a later selfie-view display, and convert
-  # the BGR image to RGB.
-  image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
-  # To improve performance, optionally mark the image as not writeable to
-  # pass by reference.
-  image.flags.writeable = False
-  results = face_mesh.process(image)
-
-  # Draw the face mesh annotations on the image.
-  image.flags.writeable = True
-  image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-  if results.multi_face_landmarks:
+    # Print and draw face mesh landmarks on the image.
+    if not results.multi_face_landmarks:
+      continue
+    annotated_image = image.copy()
     for face_landmarks in results.multi_face_landmarks:
+      print('face_landmarks:', face_landmarks)
       mp_drawing.draw_landmarks(
-          image=image,
+          image=annotated_image,
           landmark_list=face_landmarks,
           connections=mp_face_mesh.FACE_CONNECTIONS,
           landmark_drawing_spec=drawing_spec,
           connection_drawing_spec=drawing_spec)
-  cv2.imshow('MediaPipe FaceMesh', image)
-  if cv2.waitKey(5) & 0xFF == 27:
-    break
-face_mesh.close()
+    cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
+
+# For webcam input:
+drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+cap = cv2.VideoCapture(0)
+with mp_face_mesh.FaceMesh(
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
+
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = face_mesh.process(image)
+
+    # Draw the face mesh annotations on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    if results.multi_face_landmarks:
+      for face_landmarks in results.multi_face_landmarks:
+        mp_drawing.draw_landmarks(
+            image=image,
+            landmark_list=face_landmarks,
+            connections=mp_face_mesh.FACE_CONNECTIONS,
+            landmark_drawing_spec=drawing_spec,
+            connection_drawing_spec=drawing_spec)
+    cv2.imshow('MediaPipe FaceMesh', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
 cap.release()
 ```
 
 ### JavaScript Solution API
 
 Please first see general [introduction](../getting_started/javascript.md) on
-MediaPipe in JavaScript, then learn more in the companion [web demo] and the
-following usage example.
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
 
 Supported configuration options:
 
@@ -503,7 +502,5 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
     [OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
     [UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
 *   [Models and model cards](./models.md#face_mesh)
-
-[Colab]:https://mediapipe.page.link/face_mesh_py_colab
-
-[web demo]:https://code.mediapipe.dev/codepen/face_mesh
+*   [Web demo](https://code.mediapipe.dev/codepen/face_mesh)
+*   [Python Colab](https://mediapipe.page.link/face_mesh_py_colab)

docs/solutions/hands.md

@@ -91,13 +91,14 @@ To detect initial hand locations, we designed a
 mobile real-time uses in a manner similar to the face detection model in
 [MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
 task: our
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite) has
-to work across a variety of hand sizes with a large scale span (~20x) relative
-to the image frame and be able to detect occluded and self-occluded hands.
-Whereas faces have high contrast patterns, e.g., in the eye and mouth region,
-the lack of such features in hands makes it comparatively difficult to detect
-them reliably from their visual features alone. Instead, providing additional
-context, like arm, body, or person features, aids accurate hand localization.
+[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite)
+has to work across a variety of hand sizes with a large scale span (~20x)
+relative to the image frame and be able to detect occluded and self-occluded
+hands. Whereas faces have high contrast patterns, e.g., in the eye and mouth
+region, the lack of such features in hands makes it comparatively difficult to
+detect them reliably from their visual features alone. Instead, providing
+additional context, like arm, body, or person features, aids accurate hand
+localization.
 
 Our method addresses the above challenges using different strategies. First, we
 train a palm detector instead of a hand detector, since estimating bounding
@@ -119,7 +120,7 @@ just 86.22%.
 ### Hand Landmark Model
 
 After the palm detection over the whole image our subsequent hand landmark
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite)
+[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite)
 performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
 the detected hand regions via regression, that is direct coordinate prediction.
 The model learns a consistent internal hand pose representation and is robust
@@ -136,11 +137,9 @@ to the corresponding 3D coordinates.
 :--------------------------------------------------------: |
 *Fig 2. 21 hand landmarks.*                                |
 
-| ![hand_crops.png](../images/mobile/hand_crops.png)                          |
-| :-------------------------------------------------------------------------: |
-| *Fig 3. Top: Aligned hand crops passed to the tracking network with ground  |
-: truth annotation. Bottom\: Rendered synthetic hand images with ground truth :
-: annotation.*                                                                :
+![hand_crops.png](../images/mobile/hand_crops.png)                          |
+:-------------------------------------------------------------------------: |
+*Fig 3. Top: Aligned hand crops passed to the tracking network with ground truth annotation. Bottom: Rendered synthetic hand images with ground truth annotation.* |
 
 ## Solution APIs
 
@@ -206,8 +205,8 @@ is not the case, please swap the handedness output in the application.
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
-install MediaPipe Python package, then learn more in the companion [Colab] and
-the following usage example.
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
 
 Supported configuration options:
 
@@ -223,74 +222,73 @@ mp_drawing = mp.solutions.drawing_utils
 mp_hands = mp.solutions.hands
 
 # For static images:
-hands = mp_hands.Hands(
+with mp_hands.Hands(
     static_image_mode=True,
     max_num_hands=2,
-    min_detection_confidence=0.5)
-for idx, file in enumerate(file_list):
-  # Read an image, flip it around y-axis for correct handedness output (see
-  # above).
-  image = cv2.flip(cv2.imread(file), 1)
-  # Convert the BGR image to RGB before processing.
-  results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+    min_detection_confidence=0.5) as hands:
+  for idx, file in enumerate(file_list):
+    # Read an image, flip it around y-axis for correct handedness output (see
+    # above).
+    image = cv2.flip(cv2.imread(file), 1)
+    # Convert the BGR image to RGB before processing.
+    results = hands.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
 
-  # Print handedness and draw hand landmarks on the image.
-  print('Handedness:', results.multi_handedness)
-  if not results.multi_hand_landmarks:
-    continue
-  image_hight, image_width, _ = image.shape
-  annotated_image = image.copy()
-  for hand_landmarks in results.multi_hand_landmarks:
-    print('hand_landmarks:', hand_landmarks)
-    print(
-        f'Index finger tip coordinates: (',
-        f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
-        f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_hight})'
-    )
-    mp_drawing.draw_landmarks(
-        annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
-  cv2.imwrite(
-      '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
-hands.close()
+    # Print handedness and draw hand landmarks on the image.
+    print('Handedness:', results.multi_handedness)
+    if not results.multi_hand_landmarks:
+      continue
+    image_height, image_width, _ = image.shape
+    annotated_image = image.copy()
+    for hand_landmarks in results.multi_hand_landmarks:
+      print('hand_landmarks:', hand_landmarks)
+      print(
+          f'Index finger tip coordinates: (',
+          f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].x * image_width}, '
+          f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
+      )
+      mp_drawing.draw_landmarks(
+          annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+    cv2.imwrite(
+        '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
 
 # For webcam input:
-hands = mp_hands.Hands(
-    min_detection_confidence=0.5, min_tracking_confidence=0.5)
 cap = cv2.VideoCapture(0)
-while cap.isOpened():
-  success, image = cap.read()
-  if not success:
-    print("Ignoring empty camera frame.")
-    # If loading a video, use 'break' instead of 'continue'.
-    continue
+with mp_hands.Hands(
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as hands:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
 
-  # Flip the image horizontally for a later selfie-view display, and convert
-  # the BGR image to RGB.
-  image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
-  # To improve performance, optionally mark the image as not writeable to
-  # pass by reference.
-  image.flags.writeable = False
-  results = hands.process(image)
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = hands.process(image)
 
-  # Draw the hand annotations on the image.
-  image.flags.writeable = True
-  image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-  if results.multi_hand_landmarks:
-    for hand_landmarks in results.multi_hand_landmarks:
-      mp_drawing.draw_landmarks(
-          image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
-  cv2.imshow('MediaPipe Hands', image)
-  if cv2.waitKey(5) & 0xFF == 27:
-    break
-hands.close()
+    # Draw the hand annotations on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    if results.multi_hand_landmarks:
+      for hand_landmarks in results.multi_hand_landmarks:
+        mp_drawing.draw_landmarks(
+            image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+    cv2.imshow('MediaPipe Hands', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
 cap.release()
 ```
 
 ### JavaScript Solution API
 
 Please first see general [introduction](../getting_started/javascript.md) on
-MediaPipe in JavaScript, then learn more in the companion [web demo] and a
-[fun application], and the following usage example.
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and a [fun application], and the following usage example.
 
 Supported configuration options:
 
@@ -425,8 +423,6 @@ it, in the graph file modify the option of `ConstantSidePacketCalculator`.
     [MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
     ([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
 *   [Models and model cards](./models.md#hands)
-
-[Colab]:https://mediapipe.page.link/hands_py_colab
-
-[web demo]:https://code.mediapipe.dev/codepen/hands
-[fun application]:https://code.mediapipe.dev/codepen/defrost
+*   [Web demo](https://code.mediapipe.dev/codepen/hands)
+*   [Fun application](https://code.mediapipe.dev/codepen/defrost)
+*   [Python Colab](https://mediapipe.page.link/hands_py_colab)

docs/solutions/holistic.md

@@ -201,8 +201,8 @@ A list of 21 hand landmarks on the right hand, in the same representation as
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
-install MediaPipe Python package, then learn more in the companion [Colab] and
-the following usage example.
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
 
 Supported configuration options:
 
@@ -219,74 +219,75 @@ mp_drawing = mp.solutions.drawing_utils
 mp_holistic = mp.solutions.holistic
 
 # For static images:
-holistic = mp_holistic.Holistic(static_image_mode=True)
-for idx, file in enumerate(file_list):
-  image = cv2.imread(file)
-  image_hight, image_width, _ = image.shape
-  # Convert the BGR image to RGB before processing.
-  results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+with mp_holistic.Holistic(static_image_mode=True) as holistic:
+  for idx, file in enumerate(file_list):
+    image = cv2.imread(file)
+    image_height, image_width, _ = image.shape
+    # Convert the BGR image to RGB before processing.
+    results = holistic.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
 
-  if results.pose_landmarks:
-    print(
-        f'Nose coordinates: ('
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_hight})'
-    )
-  # Draw pose, left and right hands, and face landmarks on the image.
-  annotated_image = image.copy()
-  mp_drawing.draw_landmarks(
-      annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
-  cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
-holistic.close()
+    if results.pose_landmarks:
+      print(
+          f'Nose coordinates: ('
+          f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
+          f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
+      )
+    # Draw pose, left and right hands, and face landmarks on the image.
+    annotated_image = image.copy()
+    mp_drawing.draw_landmarks(
+        annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
+    mp_drawing.draw_landmarks(
+        annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+    mp_drawing.draw_landmarks(
+        annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+    # Use mp_holistic.UPPER_BODY_POSE_CONNECTIONS for drawing below when
+    # upper_body_only is set to True.
+    mp_drawing.draw_landmarks(
+        annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
+    cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
-holistic = mp_holistic.Holistic(
-    min_detection_confidence=0.5, min_tracking_confidence=0.5)
 cap = cv2.VideoCapture(0)
-while cap.isOpened():
-  success, image = cap.read()
-  if not success:
-    print("Ignoring empty camera frame.")
-    # If loading a video, use 'break' instead of 'continue'.
-    continue
+with mp_holistic.Holistic(
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as holistic:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
 
-  # Flip the image horizontally for a later selfie-view display, and convert
-  # the BGR image to RGB.
-  image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
-  # To improve performance, optionally mark the image as not writeable to
-  # pass by reference.
-  image.flags.writeable = False
-  results = holistic.process(image)
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = holistic.process(image)
 
-  # Draw landmark annotation on the image.
-  image.flags.writeable = True
-  image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-  mp_drawing.draw_landmarks(
-      image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
-  mp_drawing.draw_landmarks(
-      image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
-  cv2.imshow('MediaPipe Holistic', image)
-  if cv2.waitKey(5) & 0xFF == 27:
-    break
-holistic.close()
+    # Draw landmark annotation on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    mp_drawing.draw_landmarks(
+        image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
+    mp_drawing.draw_landmarks(
+        image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+    mp_drawing.draw_landmarks(
+        image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+    mp_drawing.draw_landmarks(
+        image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
+    cv2.imshow('MediaPipe Holistic', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
 cap.release()
 ```
 
 ### JavaScript Solution API
 
 Please first see general [introduction](../getting_started/javascript.md) on
-MediaPipe in JavaScript, then learn more in the companion [web demo] and the
-following usage example.
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
 
 Supported configuration options:
 
@@ -405,9 +406,7 @@ on how to build MediaPipe examples.
 ## Resources
 
 *   Google AI Blog:
-    [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction on Device](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
+    [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
 *   [Models and model cards](./models.md#holistic)
-
-[Colab]:https://mediapipe.page.link/holistic_py_colab
-
-[web demo]:https://code.mediapipe.dev/codepen/holistic
+*   [Web demo](https://code.mediapipe.dev/codepen/holistic)
+*   [Python Colab](https://mediapipe.page.link/holistic_py_colab)

docs/solutions/instant_motion_tracking.md

@@ -117,6 +117,25 @@ Please first see general instructions for
 * Android target (or download prebuilt [ARM64 APK](https://drive.google.com/file/d/1KnaBBoKpCHR73nOBJ4fL_YdWVTAcwe6L/view?usp=sharing)):
 [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking:instantmotiontracking`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD)
 
+* Assets rendered by the [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) must be preprocessed into an OpenGL-ready custom .uuu format. This can be done
+for user assets as follows:
+> First run
+>
+> ```shell
+> ./mediapipe/graphs/object_detection_3d/obj_parser/obj_cleanup.sh [INPUT_DIR] [INTERMEDIATE_OUTPUT_DIR]
+> ```
+> and then run
+>
+> ```build
+> bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
+> ```
+> INPUT_DIR should be the folder with initial asset .obj files to be processed,
+> and OUTPUT_DIR is the folder where the processed asset .uuu file will be placed.
+>
+> Note: ObjParser combines all .obj files found in the given directory into a
+> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
+> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
+
 ## Resources
 
 *   Google Developers Blog:

docs/solutions/models.md

@@ -41,8 +41,9 @@ nav_order: 30
     [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
 *   Hand landmark model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
+    [TFLite model (sparse)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_sparse.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-*   [Model card](https://mediapipe.page.link/handmc)
+*   [Model card](https://mediapipe.page.link/handmc), [Model card (sparse)](https://mediapipe.page.link/handmc-sparse)
 
 ### [Pose](https://google.github.io/mediapipe/solutions/pose)
 
@@ -73,12 +74,12 @@ nav_order: 30
 
 ### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
 
-*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
-*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
-*   [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_camera.tflite)
-*   [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_cup.tflite)
-*   [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers_1stage.tflite)
-*   [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair_1stage.tflite)
+*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers.tflite)
+*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair.tflite)
+*   [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_camera.tflite)
+*   [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_cup.tflite)
+*   [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite)
+*   [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite)
 *   [Model card](https://mediapipe.page.link/objectron-mc)
 
 ### [KNIFT](https://google.github.io/mediapipe/solutions/knift)

docs/solutions/objectron.md

@@ -186,18 +186,189 @@ trained our 3D object detection models. The technical details of the Objectron
 dataset, including usage and tutorials, are available on
 the [dataset website](https://github.com/google-research-datasets/Objectron/).
 
+## Solution APIs
+
+### Cross-platform Configuration Options
+
+Naming style and availability may differ slightly across platforms/languages.
+
+#### static_image_mode
+
+If set to `false`, the solution treats the input images as a video stream. It
+will try to detect objects in the very first images, and upon successful
+detection further localizes the 3D bounding box landmarks. In subsequent images,
+once all [max_num_objects](#max_num_objects) objects are detected and the
+corresponding 3D bounding box landmarks are localized, it simply tracks those
+landmarks without invoking another detection until it loses track of any of the
+objects. This reduces latency and is ideal for processing video frames. If set
+to `true`, object detection runs every input image, ideal for processing a batch
+of static, possibly unrelated, images. Default to `false`.
+
+#### max_num_objects
+
+Maximum number of objects to detect. Default to `5`.
+
+#### min_detection_confidence
+
+Minimum confidence value (`[0.0, 1.0]`) from the object-detection model for the
+detection to be considered successful. Default to `0.5`.
+
+#### min_tracking_confidence
+
+Minimum confidence value (`[0.0, 1.0]`) from the landmark-tracking model for the
+3D bounding box landmarks to be considered tracked successfully, or otherwise
+object detection will be invoked automatically on the next input image. Setting
+it to a higher value can increase robustness of the solution, at the expense of
+a higher latency. Ignored if [static_image_mode](#static_image_mode) is `true`,
+where object detection simply runs on every image. Default to `0.99`.
+
+#### model_name
+
+Name of the model to use for predicting 3D bounding box landmarks. Currently supports
+`{'Shoe', 'Chair', 'Cup', 'Camera'}`.
+
+#### focal_length
+
+Camera focal length `(fx, fy)`, by default is defined in
+[NDC space](#ndc-space). To use focal length `(fx_pixel, fy_pixel)` in
+[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
+image_height)` to enable conversions inside the API. For further details about
+NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
+
+#### principal_point
+
+Camera principal point `(px, py)`, by default is defined in
+[NDC space](#ndc-space). To use principal point `(px_pixel, py_pixel)` in
+[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
+image_height)` to enable conversions inside the API. For further details about
+NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
+
+#### image_size
+
+(**Optional**) size `(image_width, image_height)` of the input image, **ONLY**
+needed when use `focal_length` and `principal_point` in pixel space.
+
+### Output
+
+<!-- Naming style may differ slightly across platforms/languages. -->
+
+#### detected_objects
+
+A list of detected 3D bounding box. Each 3D bounding box consists of the
+following:
+
+*   `landmarks_2d` : 2D landmarks of the object's 3D bounding box. The landmark
+    coordinates are normalized to `[0.0, 1.0]` by the image width and height
+    respectively.
+
+*   `landmarks_3d` : 3D landmarks of the object's 3D bounding box. The landmark
+    coordinates are represented in [camera coordinate](#camera-coordinate)
+    frame.
+
+*   `rotation` : rotation matrix from object coordinate frame to camera
+    coordinate frame.
+
+*   `translation` : translation vector from object coordinate frame to camera
+    coordinate frame.
+
+*   `scale` : relative scale of the object along `x`, `y` and `z` directions.
+
+## Python Solution API
+
+Please first follow general [instructions](../getting_started/python.md) to
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
+
+Supported configuration options:
+
+*   [static_image_mode](#static_image_mode)
+*   [max_num_objects](#max_num_objects)
+*   [min_detection_confidence](#min_detection_confidence)
+*   [min_tracking_confidence](#min_tracking_confidence)
+*   [model_name](#model_name)
+*   [focal_length](#focal_length)
+*   [principal_point](#principal_point)
+*   [image_size](#image_size)
+
+```python
+import cv2
+import mediapipe as mp
+mp_drawing = mp.solutions.drawing_utils
+mp_objectron = mp.solutions.objectron
+
+# For static images:
+with mp_objectron.Objectron(static_image_mode=True,
+                            max_num_objects=5,
+                            min_detection_confidence=0.5,
+                            model_name='Shoe') as objectron:
+  for idx, file in enumerate(file_list):
+    image = cv2.imread(file)
+    # Convert the BGR image to RGB and process it with MediaPipe Objectron.
+    results = objectron.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+
+    # Draw box landmarks.
+    if not results.detected_objects:
+      print(f'No box landmarks detected on {file}')
+      continue
+    print(f'Box landmarks of {file}:')
+    annotated_image = image.copy()
+    for detected_object in results.detected_objects:
+      mp_drawing.draw_landmarks(
+          annotated_image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
+      mp_drawing.draw_axis(annotated_image, detected_object.rotation,
+                           detected_object.translation)
+      cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
+
+# For webcam input:
+cap = cv2.VideoCapture(0)
+with mp_objectron.Objectron(static_image_mode=False,
+                            max_num_objects=5,
+                            min_detection_confidence=0.5,
+                            min_tracking_confidence=0.99,
+                            model_name='Shoe') as objectron:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
+
+    # Convert the BGR image to RGB.
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = objectron.process(image)
+
+    # Draw the box landmarks on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    if results.detected_objects:
+        for detected_object in results.detected_objects:
+            mp_drawing.draw_landmarks(
+              image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
+            mp_drawing.draw_axis(image, detected_object.rotation,
+                                 detected_object.translation)
+    cv2.imshow('MediaPipe Objectron', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
+cap.release()
+```
+
 ## Example Apps
 
 Please first see general instructions for
-[Android](../getting_started/android.md) and [iOS](../getting_started/ios.md) on
-how to build MediaPipe examples.
+[Android](../getting_started/android.md), [iOS](../getting_started/ios.md), and
+[desktop](../getting_started/cpp.md) on how to build MediaPipe examples.
 
 Note: To visualize a graph, copy the graph and paste it into
 [MediaPipe Visualizer](https://viz.mediapipe.dev/). For more information on how
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Two-stage Objectron
+### Mobile
+
+#### Two-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -235,7 +406,7 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
-### Single-stage Objectron
+#### Single-stage Objectron
 
 *   Graph:
     [`mediapipe/graphs/object_detection_3d/object_occlusion_tracking_1stage.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/object_occlusion_tracking.pbtxt)
@@ -259,15 +430,141 @@ to visualize its associated subgraphs, please see
 
 *   iOS target: Not available
 
+#### Assets
+
+Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc) using a parsing of the sequenced .obj file
+ format into a custom .uuu format. This can be done for user assets as follows:
+> First run
+>
+> ```shell
+> ./mediapipe/graphs/object_detection_3d/obj_parser/obj_cleanup.sh [INPUT_DIR] [INTERMEDIATE_OUTPUT_DIR]
+> ```
+> and then run
+>
+> ```build
+> bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
+> ```
+> INPUT_DIR should be the folder with initial asset .obj files to be processed,
+> and OUTPUT_DIR is the folder where the processed asset .uuu file will be placed.
+>
+> Note: ObjParser combines all .obj files found in the given directory into a
+> single .uuu animation file, using the order given by sorting the filenames alphanumerically. Also the ObjParser directory inputs must be given as
+> absolute paths, not relative paths. See parser utility library at [`mediapipe/graphs/object_detection_3d/obj_parser/`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection_3d/obj_parser/) for more details.
+
+
+### Desktop
+
+To build the application, run:
+
+```bash
+bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu
+```
+
+To run the application, replace `<input video path>` and `<output video path>`
+in the command below with your own paths, and `<landmark model path>` and
+`<allowed labels>` with the following:
+
+Category | `<landmark model path>`                                                     | `<allowed labels>`
+:------- | :-------------------------------------------------------------------------- | :-----------------
+Shoe     | mediapipe/modules/objectron/object_detection_3d_sneakers.tflite | Footwear
+Chair    | mediapipe/modules/objectron/object_detection_3d_chair.tflite    | Chair
+Cup      | mediapipe/modules/objectron/object_detection_3d_cup.tflite      | Mug
+Camera   | mediapipe/modules/objectron/object_detection_3d_camera.tflite   | Camera
+
+```
+GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \
+  --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \
+  --input_side_packets=input_video_path=<input video path>,output_video_path=<output video path>,box_landmark_model_path=<landmark model path>,allowed_labels=<allowed labels>
+```
+
+## Coordinate Systems
+
+### Object Coordinate
+
+Each object has its object coordinate frame. We use the below object coordinate
+definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
+origin is at the center of the 3D bounding box.
+
+![box_coordinate.svg](../images/box_coordinate.svg)
+
+### Camera Coordinate
+
+A 3D object is parameterized by its `scale` and `rotation`, `translation` with
+regard to the camera coordinate frame. In this API we use the below camera
+coordinate definition, with `+x` pointing right, `+y` pointing up and `-z`
+pointing to the scene.
+
+![camera_coordinate.svg](../images/camera_coordinate.svg)
+
+To work with box landmarks, one can first derive landmark coordinates in object
+frame by scaling a origin centered unit box with `scale`, then transform to
+camera frame by applying `rotation` and `translation`:
+
+```
+landmarks_3d = rotation * scale * unit_box + translation
+```
+
+### NDC Space
+
+In this API we use
+[NDC(normalized device coordinates)](http://www.songho.ca/opengl/gl_projectionmatrix.html)
+as an intermediate space when projecting points from 3D to 2D. In NDC space,
+`x`, `y` are confined to `[-1, 1]`.
+
+![ndc_coordinate.svg](../images/ndc_coordinate.svg)
+
+By default the camera parameters `(fx, fy)` and `(px, py)` are defined in NDC
+space. Given `(X, Y, Z)` of 3D points in camera coordinate, one can project 3D
+points to NDC space as follows:
+
+```
+x_ndc = -fx * X / Z + px
+y_ndc = -fy * Y / Z + py
+z_ndc = 1 / Z
+```
+
+### Pixel Space
+
+In this API we set upper-left coner of an image as the origin of pixel
+coordinate. One can convert from NDC to pixel space as follows:
+
+```
+x_pixel = (1 + x_ndc) / 2.0 * image_width
+y_pixel = (1 - y_ndc) / 2.0 * image_height
+```
+
+Alternatively one can directly project from camera coordinate to pixel
+coordinate with camera parameters `(fx_pixel, fy_pixel)` and `(px_pixel,
+py_pixel)` defined in pixel space as follows:
+
+```
+x_pixel = -fx_pixel * X / Z + px_pixel
+y_pixel =  fy_pixel * Y / Z + py_pixel
+```
+
+Conversion of camera parameters from pixel space to NDC space:
+
+```
+fx = fx_pixel * 2.0 / image_width
+fy = fy_pixel * 2.0 / image_height
+```
+
+```
+px = -px_pixel * 2.0 / image_width  + 1.0
+py = -py_pixel * 2.0 / image_height + 1.0
+```
+
 ## Resources
 
 *   Google AI Blog:
     [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
 *   Google AI Blog:
     [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
+*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
 *   Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
     Shape Supervision](https://arxiv.org/abs/2003.03522)
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
-    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
+    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
 *   [Models and model cards](./models.md#objectron)
+*   [Python Colab](https://mediapipe.page.link/objectron_py_colab)

docs/solutions/pose.md

@@ -2,6 +2,8 @@
 layout: default
 title: Pose
 parent: Solutions
+has_children: true
+has_toc: false
 nav_order: 5
 ---
 
@@ -21,13 +23,14 @@ nav_order: 5
 ## Overview
 
 Human pose estimation from video plays a critical role in various applications
-such as quantifying physical exercises, sign language recognition, and full-body
-gesture control. For example, it can form the basis for yoga, dance, and fitness
-applications. It can also enable the overlay of digital content and information
-on top of the physical world in augmented reality.
+such as [quantifying physical exercises](./pose_classification.md), sign
+language recognition, and full-body gesture control. For example, it can form
+the basis for yoga, dance, and fitness applications. It can also enable the
+overlay of digital content and information on top of the physical world in
+augmented reality.
 
 MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
-33 2D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
+33 3D landmarks on the whole body (or 25 upper-body landmarks) from RGB video
 frames utilizing our
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 research that also powers the
@@ -35,7 +38,7 @@ research that also powers the
 Current state-of-the-art approaches rely primarily on powerful desktop
 environments for inference, whereas our method achieves real-time performance on
 most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
-[python](#python) and even on the [web](#web).
+[python](#python-solution-api) and even on the [web](#javascript-solution-api).
 
 ![pose_tracking_upper_body_example.gif](../images/mobile/pose_tracking_upper_body_example.gif) |
 :--------------------------------------------------------------------------------------------: |
@@ -92,7 +95,7 @@ hip midpoints.
 :----------------------------------------------------------------------------------------------------: |
 *Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
 
-### Pose Landmark Model (BlazePose Tracker)
+### Pose Landmark Model (BlazePose GHUM 3D)
 
 The landmark model in MediaPipe Pose comes in two versions: a full-body model
 that predicts the location of 33 pose landmarks (see figure below), and an
@@ -163,16 +166,21 @@ A list of pose landmarks. Each lanmark consists of the following:
 
 *   `x` and `y`: Landmark coordinates normalized to `[0.0, 1.0]` by the image
     width and height respectively.
-*   `z`: Should be discarded as currently the model is not fully trained to
-    predict depth, but this is something on the roadmap.
+*   `z`: Represents the landmark depth with the depth at the midpoint of hips
+    being the origin, and the smaller the value the closer the landmark is to
+    the camera. The magnitude of `z` uses roughly the same scale as `x`.
+
+    Note: `z` is predicted only in full-body mode, and should be discarded when
+    [upper_body_only](#upper_body_only) is `true`.
+
 *   `visibility`: A value in `[0.0, 1.0]` indicating the likelihood of the
     landmark being visible (present and not occluded) in the image.
 
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
-install MediaPipe Python package, then learn more in the companion [Colab] and
-the following usage example.
+install MediaPipe Python package, then learn more in the companion
+[Python Colab](#resources) and the following usage example.
 
 Supported configuration options:
 
@@ -189,64 +197,65 @@ mp_drawing = mp.solutions.drawing_utils
 mp_pose = mp.solutions.pose
 
 # For static images:
-pose = mp_pose.Pose(
-    static_image_mode=True, min_detection_confidence=0.5)
-for idx, file in enumerate(file_list):
-  image = cv2.imread(file)
-  image_hight, image_width, _ = image.shape
-  # Convert the BGR image to RGB before processing.
-  results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+with mp_pose.Pose(
+    static_image_mode=True, min_detection_confidence=0.5) as pose:
+  for idx, file in enumerate(file_list):
+    image = cv2.imread(file)
+    image_height, image_width, _ = image.shape
+    # Convert the BGR image to RGB before processing.
+    results = pose.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
 
-  if not results.pose_landmarks:
-    continue
-  print(
-      f'Nose coordinates: ('
-      f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
-      f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_hight})'
-  )
-  # Draw pose landmarks on the image.
-  annotated_image = image.copy()
-  mp_drawing.draw_landmarks(
-      annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
-  cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
-pose.close()
+    if not results.pose_landmarks:
+      continue
+    print(
+        f'Nose coordinates: ('
+        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
+        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
+    )
+    # Draw pose landmarks on the image.
+    annotated_image = image.copy()
+    # Use mp_pose.UPPER_BODY_POSE_CONNECTIONS for drawing below when
+    # upper_body_only is set to True.
+    mp_drawing.draw_landmarks(
+        annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
+    cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
-pose = mp_pose.Pose(
-    min_detection_confidence=0.5, min_tracking_confidence=0.5)
 cap = cv2.VideoCapture(0)
-while cap.isOpened():
-  success, image = cap.read()
-  if not success:
-    print("Ignoring empty camera frame.")
-    # If loading a video, use 'break' instead of 'continue'.
-    continue
+with mp_pose.Pose(
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as pose:
+  while cap.isOpened():
+    success, image = cap.read()
+    if not success:
+      print("Ignoring empty camera frame.")
+      # If loading a video, use 'break' instead of 'continue'.
+      continue
 
-  # Flip the image horizontally for a later selfie-view display, and convert
-  # the BGR image to RGB.
-  image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
-  # To improve performance, optionally mark the image as not writeable to
-  # pass by reference.
-  image.flags.writeable = False
-  results = pose.process(image)
+    # Flip the image horizontally for a later selfie-view display, and convert
+    # the BGR image to RGB.
+    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
+    # To improve performance, optionally mark the image as not writeable to
+    # pass by reference.
+    image.flags.writeable = False
+    results = pose.process(image)
 
-  # Draw the pose annotation on the image.
-  image.flags.writeable = True
-  image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-  mp_drawing.draw_landmarks(
-      image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
-  cv2.imshow('MediaPipe Pose', image)
-  if cv2.waitKey(5) & 0xFF == 27:
-    break
-pose.close()
+    # Draw the pose annotation on the image.
+    image.flags.writeable = True
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+    mp_drawing.draw_landmarks(
+        image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
+    cv2.imshow('MediaPipe Pose', image)
+    if cv2.waitKey(5) & 0xFF == 27:
+      break
 cap.release()
 ```
 
 ### JavaScript Solution API
 
 Please first see general [introduction](../getting_started/javascript.md) on
-MediaPipe in JavaScript, then learn more in the companion [web demo] and the
-following usage example.
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
 
 Supported configuration options:
 
@@ -387,7 +396,5 @@ on how to build MediaPipe examples.
     [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
     ([presentation](https://youtu.be/YPpUOTRn5tA))
 *   [Models and model cards](./models.md#pose)
-
-[Colab]:https://mediapipe.page.link/pose_py_colab
-
-[web demo]:https://code.mediapipe.dev/codepen/pose
+*   [Web demo](https://code.mediapipe.dev/codepen/pose)
+*   [Python Colab](https://mediapipe.page.link/pose_py_colab)

docs/solutions/pose_classification.md

@@ -0,0 +1,145 @@
+---
+layout: default
+title: Pose Classification
+parent: Pose
+grand_parent: Solutions
+nav_order: 1
+---
+
+# Pose Classification
+{: .no_toc }
+
+<details close markdown="block">
+  <summary>
+    Table of contents
+  </summary>
+  {: .text-delta }
+1. TOC
+{:toc}
+</details>
+---
+
+## Overview
+
+One of the applications
+[BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
+can enable is fitness. More specifically - pose classification and repetition
+counting. In this section we'll provide basic guidance on building a custom pose
+classifier with the help of [Colabs](#colabs) and wrap it in a simple fitness
+demo within
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
+Push-ups and squats are used for demonstration purposes as the most common
+exercises.
+
+![pose_classification_pushups_and_squats.gif](../images/mobile/pose_classification_pushups_and_squats.gif) |
+:--------------------------------------------------------------------------------------------------------: |
+*Fig 1. Pose classification and repetition counting with MediaPipe Pose.*                                  |
+
+We picked the
+[k-nearest neighbors algorithm](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm)
+(k-NN) as the classifier. It's simple and easy to start with. The algorithm
+determines the object's class based on the closest samples in the training set.
+
+**To build it, one needs to:**
+
+1.  Collect image samples of the target exercises and run pose prediction on
+    them,
+2.  Convert obtained pose landmarks to a representation suitable for the k-NN
+    classifier and form a training set using these [Colabs](#colabs),
+3.  Perform the classification itself followed by repetition counting (e.g., in
+    the
+    [ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app)).
+
+## Training Set
+
+To build a good classifier appropriate samples should be collected for the
+training set: about a few hundred samples for each terminal state of each
+exercise (e.g., "up" and "down" positions for push-ups). It's important that
+collected samples cover different camera angles, environment conditions, body
+shapes, and exercise variations.
+
+![pose_classification_pushups_un_and_down_samples.jpg](../images/mobile/pose_classification_pushups_un_and_down_samples.jpg) |
+:--------------------------------------------------------------------------------------------------------------------------: |
+*Fig 2. Two terminal states of push-ups.*                                                                                    |
+
+To transform samples into a k-NN classifier training set, both
+[`Pose Classification Colab (Basic)`] and
+[`Pose Classification Colab (Extended)`] could be used. They use the
+[Python Solution API](./pose.md#python-solution-api) to run the BlazePose models
+on given images and dump predicted pose landmarks to a CSV file. Additionally,
+the [`Pose Classification Colab (Extended)`] provides useful tools to find
+outliers (e.g., wrongly predicted poses) and underrepresented classes (e.g., not
+covering all camera angles) by classifying each sample against the entire
+training set. After that, you'll be able to test the classifier on an arbitrary
+video right in the Colab.
+
+## Classification
+
+Code of the classifier is available both in the
+[`Pose Classification Colab (Extended)`] and in the
+[ML Kit quickstart app](https://developers.google.com/ml-kit/vision/pose-detection/classifying-poses#4_integrate_with_the_ml_kit_quickstart_app).
+Please refer to them for details of the approach described below.
+
+The k-NN algorithm used for pose classification requires a feature vector
+representation of each sample and a metric to compute the distance between two
+such vectors to find the nearest pose samples to a target one.
+
+To convert pose landmarks to a feature vector, we use pairwise distances between
+predefined lists of pose joints, such as distances between wrist and shoulder,
+ankle and hip, and two wrists. Since the algorithm relies on distances, all
+poses are normalized to have the same torso size and vertical torso orientation
+before the conversion.
+
+![pose_classification_pairwise_distances.png](../images/mobile/pose_classification_pairwise_distances.png) |
+:--------------------------------------------------------------------------------------------------------: |
+*Fig 3. Main pairwise distances used for the pose feature vector.*                                         |
+
+To get a better classification result, k-NN search is invoked twice with
+different distance metrics:
+
+*   First, to filter out samples that are almost the same as the target one but
+    have only a few different values in the feature vector (which means
+    differently bent joints and thus other pose class), minimum per-coordinate
+    distance is used as distance metric,
+*   Then average per-coordinate distance is used to find the nearest pose
+    cluster among those from the first search.
+
+Finally, we apply
+[exponential moving average](https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average)
+(EMA) smoothing to level any noise from pose prediction or classification. To do
+that, we search not only for the nearest pose cluster, but we calculate a
+probability for each of them and use it for smoothing over time.
+
+## Repetition Counting
+
+To count the repetitions, the algorithm monitors the probability of a target
+pose class. Let's take push-ups with its "up" and "down" terminal states:
+
+*   When the probability of the "down" pose class passes a certain threshold for
+    the first time, the algorithm marks that the "down" pose class is entered.
+*   Once the probability drops below the threshold, the algorithm marks that the
+    "down" pose class has been exited and increases the counter.
+
+To avoid cases when the probability fluctuates around the threshold (e.g., when
+the user pauses between "up" and "down" states) causing phantom counts, the
+threshold used to detect when the state is exited is actually slightly lower
+than the one used to detect when the state is entered. It creates an interval
+where the pose class and the counter can't be changed.
+
+## Future Work
+
+We are actively working on improving
+[BlazePose GHUM 3D](./pose.md#pose-landmark-model-blazepose-ghum-3d)'s Z
+prediction. It will allow us to use joint angles in the feature vectors, which
+are more natural and easier to configure (although distances can still be useful
+to detect touches between body parts) and to perform rotation normalization of
+poses and reduce the number of camera angles required for accurate k-NN
+classification.
+
+## Colabs
+
+*   [`Pose Classification Colab (Basic)`]
+*   [`Pose Classification Colab (Extended)`]
+
+[`Pose Classification Colab (Basic)`]: https://mediapipe.page.link/pose_classification_basic
+[`Pose Classification Colab (Extended)`]: https://mediapipe.page.link/pose_classification_extended

docs/solutions/solutions.md

@@ -18,7 +18,7 @@ has_toc: false
 
 []()                                                                                      | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
 :---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
-[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
+[Face Detection](https://google.github.io/mediapipe/solutions/face_detection)                   | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             | ✅
 [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)                             | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
 [Iris](https://google.github.io/mediapipe/solutions/iris)                                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Hands](https://google.github.io/mediapipe/solutions/hands)                                     | ✅                                                               | ✅                                                       | ✅                                                       | ✅                                                             | ✅                                                             |
@@ -28,7 +28,7 @@ has_toc: false
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                        | ✅                                                             |                                                               |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

docs/tools/tracing_and_profiling.md

@@ -41,6 +41,7 @@ profiler_config {
   trace_enabled: true
   enable_profiler: true
   trace_log_interval_count: 200
+  trace_log_path: "/sdcard/Download/"
 }
 ```
 
@@ -64,7 +65,7 @@ MediaPipe will emit data into a pre-specified directory:
 
 *   On the desktop, this will be the `/tmp` directory.
 
-*   On Android, this will be the `/sdcard` directory.
+*   On Android, this will be the external storage directory (e.g., `/storage/emulated/0/`).
 
 *   On iOS, this can be reached through XCode. Select "Window/Devices and
     Simulators" and select the "Devices" tab.
@@ -103,7 +104,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     *   Include the line below in your `AndroidManifest.xml` file.
 
         ```xml
-        <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+        <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
         ```
 
     *   Grant the permission either upon first app launch, or by going into
@@ -130,8 +131,8 @@ we record ten intervals of half a second each. This can be overridden by adding
     events to a trace log files at:
 
     ```bash
-    /sdcard/mediapipe_trace_0.binarypb
-    /sdcard/mediapipe_trace_1.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_0.binarypb
+    /storage/emulated/0/Download/mediapipe_trace_1.binarypb
     ```
 
     After every 5 sec, writing shifts to a successive trace log file, such that
@@ -139,10 +140,10 @@ we record ten intervals of half a second each. This can be overridden by adding
     trace files have been written to the device using adb shell.
 
     ```bash
-    adb shell "ls -la /sdcard/"
+    adb shell "ls -la /storage/emulated/0/Download"
     ```
 
-    On android, MediaPipe selects the external storage directory `/sdcard` for
+    On android, MediaPipe selects the external storage (e.g., `/storage/emulated/0/`) for
     trace logs. This directory can be overridden using the setting
     `trace_log_path`, like:
 
@@ -150,7 +151,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     profiler_config {
       trace_enabled: true
       enable_profiler: true
-      trace_log_path: "/sdcard/profiles/"
+      trace_log_path: "/sdcard/Download/profiles/"
     }
     ```
 
@@ -161,7 +162,7 @@ we record ten intervals of half a second each. This can be overridden by adding
 
     ```bash
     # from your terminal
-    adb pull /sdcard/mediapipe_trace_0.binarypb
+    adb pull /storage/emulated/0/Download/mediapipe_trace_0.binarypb
     # if successful you should see something like
     # /sdcard/mediapipe_trace_0.binarypb: 1 file pulled. 0.1 MB/s (6766 bytes in 0.045s)
     ```

docs/tools/visualizer.md

@@ -37,7 +37,7 @@ The graph can be modified by adding and editing code in the Editor view.
     ![New Button](../images/upload_button.png)
 
 *   Pressing the "Upload" button will prompt the user to select a local PBTXT
-    file, which will everwrite the current code within the editor.
+    file, which will overwrite the current code within the editor.
 
 *   Alternatively, code can be pasted directly into the editor window.
 

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -2,37 +2,39 @@
   "additionalFilePaths" : [
     "/BUILD",
     "mediapipe/BUILD",
-    "mediapipe/objc/BUILD",
-    "mediapipe/framework/BUILD",
-    "mediapipe/gpu/BUILD",
-    "mediapipe/objc/testing/app/BUILD",
     "mediapipe/examples/ios/common/BUILD",
-    "mediapipe/examples/ios/helloworld/BUILD",
     "mediapipe/examples/ios/facedetectioncpu/BUILD",
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
     "mediapipe/examples/ios/faceeffect/BUILD",
     "mediapipe/examples/ios/facemeshgpu/BUILD",
     "mediapipe/examples/ios/handdetectiongpu/BUILD",
     "mediapipe/examples/ios/handtrackinggpu/BUILD",
+    "mediapipe/examples/ios/helloworld/BUILD",
     "mediapipe/examples/ios/holistictrackinggpu/BUILD",
     "mediapipe/examples/ios/iristrackinggpu/BUILD",
     "mediapipe/examples/ios/objectdetectioncpu/BUILD",
     "mediapipe/examples/ios/objectdetectiongpu/BUILD",
+    "mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD",
     "mediapipe/examples/ios/posetrackinggpu/BUILD",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD"
+    "mediapipe/examples/ios/upperbodyposetrackinggpu/BUILD",
+    "mediapipe/framework/BUILD",
+    "mediapipe/gpu/BUILD",
+    "mediapipe/objc/BUILD",
+    "mediapipe/objc/testing/app/BUILD"
   ],
   "buildTargets" : [
-    "//mediapipe/examples/ios/helloworld:HelloWorldApp",
     "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
     "//mediapipe/examples/ios/faceeffect:FaceEffectApp",
     "//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
     "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
     "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
+    "//mediapipe/examples/ios/helloworld:HelloWorldApp",
     "//mediapipe/examples/ios/holistictrackinggpu:HolisticTrackingGpuApp",
     "//mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp",
     "//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
     "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
+    "//mediapipe/examples/ios/objectdetectiontrackinggpu:ObjectDetectionTrackingGpuApp",
     "//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp",
     "//mediapipe/examples/ios/upperbodyposetrackinggpu:UpperBodyPoseTrackingGpuApp",
     "//mediapipe/objc:mediapipe_framework_ios"
@@ -91,13 +93,13 @@
     "mediapipe/examples/ios",
     "mediapipe/examples/ios/common",
     "mediapipe/examples/ios/common/Base.lproj",
-    "mediapipe/examples/ios/helloworld",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
     "mediapipe/examples/ios/faceeffect",
     "mediapipe/examples/ios/faceeffect/Base.lproj",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",
+    "mediapipe/examples/ios/helloworld",
     "mediapipe/examples/ios/holistictrackinggpu",
     "mediapipe/examples/ios/iristrackinggpu",
     "mediapipe/examples/ios/objectdetectioncpu",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -9,7 +9,6 @@
   "packages" : [
     "",
     "mediapipe",
-    "mediapipe/objc",
     "mediapipe/examples/ios",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
@@ -21,8 +20,10 @@
     "mediapipe/examples/ios/iristrackinggpu",
     "mediapipe/examples/ios/objectdetectioncpu",
     "mediapipe/examples/ios/objectdetectiongpu",
+    "mediapipe/examples/ios/objectdetectiontrackinggpu",
     "mediapipe/examples/ios/posetrackinggpu",
-    "mediapipe/examples/ios/upperbodyposetrackinggpu"
+    "mediapipe/examples/ios/upperbodyposetrackinggpu",
+    "mediapipe/objc"
   ],
   "projectName" : "Mediapipe",
   "workspaceRoot" : "../.."

mediapipe/calculators/audio/BUILD

@@ -1,4 +1,4 @@
-# Copyright 2019 The MediaPipe Authors.
+# Copyright 2019, 2021 The MediaPipe Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -128,7 +128,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -147,7 +147,7 @@ cc_library(
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp/mfcc",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -167,8 +167,8 @@ cc_library(
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:resampler",
-        "@com_google_audio_tools//audio/dsp:resampler_rational_factor",
-        "@eigen_archive//:eigen",
+        "@com_google_audio_tools//audio/dsp:resampler_q",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -208,7 +208,7 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@com_google_audio_tools//audio/dsp/spectrogram",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -228,7 +228,7 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -244,6 +244,7 @@ cc_test(
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -260,7 +261,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -275,7 +276,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -295,7 +296,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:number_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -313,7 +314,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -332,7 +333,7 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -351,6 +352,6 @@ cc_test(
         "//mediapipe/framework/tool:validate_type",
         "//mediapipe/util:time_series_test_util",
         "@com_google_audio_tools//audio/dsp:signal_vector_util",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )

mediapipe/calculators/audio/audio_decoder_calculator.cc

@@ -48,17 +48,17 @@ namespace mediapipe {
 // TODO: support decoding multiple streams.
 class AudioDecoderCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  static absl::Status GetContract(CalculatorContract* cc);
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
-  mediapipe::Status Close(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
 
  private:
   std::unique_ptr<AudioDecoder> decoder_;
 };
 
-mediapipe::Status AudioDecoderCalculator::GetContract(CalculatorContract* cc) {
+absl::Status AudioDecoderCalculator::GetContract(CalculatorContract* cc) {
   cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
   if (cc->InputSidePackets().HasTag("OPTIONS")) {
     cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
@@ -67,10 +67,10 @@ mediapipe::Status AudioDecoderCalculator::GetContract(CalculatorContract* cc) {
   if (cc->Outputs().HasTag("AUDIO_HEADER")) {
     cc->Outputs().Tag("AUDIO_HEADER").SetNone();
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
+absl::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
   const std::string& input_file_path =
       cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
   const auto& decoder_options =
@@ -87,10 +87,10 @@ mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
     cc->Outputs().Tag("AUDIO_HEADER").SetHeader(Adopt(header.release()));
   }
   cc->Outputs().Tag("AUDIO_HEADER").Close();
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
+absl::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
   Packet data;
   int options_index = -1;
   auto status = decoder_->GetData(&options_index, &data);
@@ -100,7 +100,7 @@ mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
   return status;
 }
 
-mediapipe::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
+absl::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
   return decoder_->Close();
 }
 

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/flags/flag.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"

mediapipe/calculators/audio/basic_time_series_calculators.cc

@@ -38,7 +38,7 @@ static bool SafeMultiply(int x, int y, int* result) {
 }
 }  // namespace
 
-mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
+absl::Status BasicTimeSeriesCalculatorBase::GetContract(
     CalculatorContract* cc) {
   cc->Inputs().Index(0).Set<Matrix>(
       // Input stream with TimeSeriesHeader.
@@ -46,10 +46,10 @@ mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
   cc->Outputs().Index(0).Set<Matrix>(
       // Output stream with TimeSeriesHeader.
   );
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
+absl::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
   TimeSeriesHeader input_header;
   MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
@@ -57,11 +57,13 @@ mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
   auto output_header = new TimeSeriesHeader(input_header);
   MP_RETURN_IF_ERROR(MutateHeader(output_header));
   cc->Outputs().Index(0).SetHeader(Adopt(output_header));
-  return mediapipe::OkStatus();
+
+  cc->SetOffset(0);
+
+  return absl::OkStatus();
 }
 
-mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
-    CalculatorContext* cc) {
+absl::Status BasicTimeSeriesCalculatorBase::Process(CalculatorContext* cc) {
   const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
   MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
       input, cc->Inputs().Index(0).Header().Get<TimeSeriesHeader>()));
@@ -71,12 +73,12 @@ mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
       *output, cc->Outputs().Index(0).Header().Get<TimeSeriesHeader>()));
 
   cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
+absl::Status BasicTimeSeriesCalculatorBase::MutateHeader(
     TimeSeriesHeader* output_header) {
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 // Calculator to sum an input time series across channels.  This is
@@ -86,9 +88,9 @@ mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
 class SumTimeSeriesAcrossChannelsCalculator
     : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_channels(1);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -104,9 +106,9 @@ REGISTER_CALCULATOR(SumTimeSeriesAcrossChannelsCalculator);
 class AverageTimeSeriesAcrossChannelsCalculator
     : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_channels(1);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -122,7 +124,7 @@ REGISTER_CALCULATOR(AverageTimeSeriesAcrossChannelsCalculator);
 // Options proto: None.
 class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     if (output_header->num_channels() != 1) {
       return tool::StatusInvalid(
           absl::StrCat("Expected single-channel input, got ",
@@ -131,7 +133,7 @@ class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
     output_header->set_num_channels(output_header->num_samples());
     output_header->set_num_samples(1);
     output_header->set_sample_rate(output_header->packet_rate());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -160,7 +162,7 @@ REGISTER_CALCULATOR(ReverseChannelOrderCalculator);
 // Options proto: None.
 class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     const int num_input_channels = output_header->num_channels();
     const int num_input_samples = output_header->num_samples();
     RET_CHECK(num_input_channels >= 0)
@@ -174,7 +176,7 @@ class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
     output_header->set_num_channels(output_num_channels);
     output_header->set_num_samples(1);
     output_header->set_sample_rate(output_header->packet_rate());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -253,10 +255,10 @@ REGISTER_CALCULATOR(DivideByMeanAcrossChannelsCalculator);
 // Options proto: None.
 class MeanCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_samples(1);
     output_header->set_sample_rate(output_header->packet_rate());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -272,10 +274,10 @@ REGISTER_CALCULATOR(MeanCalculator);
 // Options proto: None.
 class StandardDeviationCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_samples(1);
     output_header->set_sample_rate(output_header->packet_rate());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -293,9 +295,9 @@ REGISTER_CALCULATOR(StandardDeviationCalculator);
 // Options proto: None.
 class CovarianceCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_samples(output_header->num_channels());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -313,9 +315,9 @@ REGISTER_CALCULATOR(CovarianceCalculator);
 // Options proto: None.
 class L2NormCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     output_header->set_num_channels(1);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {
@@ -385,12 +387,12 @@ REGISTER_CALCULATOR(ElementwiseSquareCalculator);
 // Options proto: None.
 class FirstHalfSlicerCalculator : public BasicTimeSeriesCalculatorBase {
  protected:
-  mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+  absl::Status MutateHeader(TimeSeriesHeader* output_header) final {
     const int num_input_samples = output_header->num_samples();
     RET_CHECK(num_input_samples >= 0)
         << "FirstHalfSlicerCalculator: num_input_samples < 0";
     output_header->set_num_samples(num_input_samples / 2);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   Matrix ProcessMatrix(const Matrix& input_matrix) final {

mediapipe/calculators/audio/basic_time_series_calculators.h

@@ -28,16 +28,16 @@ namespace mediapipe {
 
 class BasicTimeSeriesCalculatorBase : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc);
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) final;
+  absl::Status Process(CalculatorContext* cc) final;
 
  protected:
   // Open() calls this method to mutate the output stream header.  The input
   // to this function will contain a copy of the input stream header, so
   // subclasses that do not need to mutate the header do not need to override
   // it.
-  virtual mediapipe::Status MutateHeader(TimeSeriesHeader* output_header);
+  virtual absl::Status MutateHeader(TimeSeriesHeader* output_header);
 
   // Process() calls this method on each packet to compute the output matrix.
   virtual Matrix ProcessMatrix(const Matrix& input_matrix) = 0;

mediapipe/calculators/audio/mfcc_mel_calculators.cc

@@ -66,7 +66,7 @@ std::string PortableDebugString(const TimeSeriesHeader& header) {
 // rows corresponding to the new feature space).
 class FramewiseTransformCalculatorBase : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<Matrix>(
         // Sequence of Matrices, each column describing a particular time frame,
         // each row a feature dimension, with TimeSeriesHeader.
@@ -75,11 +75,11 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
         // Sequence of Matrices, each column describing a particular time frame,
         // each row a feature dimension, with TimeSeriesHeader.
     );
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) final;
+  absl::Status Process(CalculatorContext* cc) final;
 
   int num_output_channels(void) { return num_output_channels_; }
 
@@ -90,8 +90,8 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
  private:
   // Takes header and options, and sets up state including calling
   // set_num_output_channels() on the base object.
-  virtual mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-                                               CalculatorContext* cc) = 0;
+  virtual absl::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                          CalculatorContext* cc) = 0;
 
   // Takes a vector<double> corresponding to an input frame, and
   // perform the specific transformation to produce an output frame.
@@ -102,23 +102,23 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
   int num_output_channels_;
 };
 
-mediapipe::Status FramewiseTransformCalculatorBase::Open(
-    CalculatorContext* cc) {
+absl::Status FramewiseTransformCalculatorBase::Open(CalculatorContext* cc) {
   TimeSeriesHeader input_header;
   MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
-  mediapipe::Status status = ConfigureTransform(input_header, cc);
+  absl::Status status = ConfigureTransform(input_header, cc);
 
   auto output_header = new TimeSeriesHeader(input_header);
   output_header->set_num_channels(num_output_channels_);
   cc->Outputs().Index(0).SetHeader(Adopt(output_header));
 
+  cc->SetOffset(0);
+
   return status;
 }
 
-mediapipe::Status FramewiseTransformCalculatorBase::Process(
-    CalculatorContext* cc) {
+absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
   const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
   const int num_frames = input.cols();
   std::unique_ptr<Matrix> output(new Matrix(num_output_channels_, num_frames));
@@ -145,7 +145,7 @@ mediapipe::Status FramewiseTransformCalculatorBase::Process(
   }
   cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 // Calculator wrapper around the dsp/mfcc/mfcc.cc routine.
@@ -170,13 +170,13 @@ mediapipe::Status FramewiseTransformCalculatorBase::Process(
 // }
 class MfccCalculator : public FramewiseTransformCalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     return FramewiseTransformCalculatorBase::GetContract(cc);
   }
 
  private:
-  mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-                                       CalculatorContext* cc) override {
+  absl::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                  CalculatorContext* cc) override {
     MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
     mfcc_.reset(new audio_dsp::Mfcc());
     int input_length = header.num_channels();
@@ -194,7 +194,7 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
     // audio_dsp::MelFilterBank needs to know this to
     // correctly interpret the spectrogram bins.
     if (!header.has_audio_sample_rate()) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
                        PortableDebugString(header)));
     }
@@ -203,10 +203,10 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
         mfcc_->Initialize(input_length, header.audio_sample_rate());
 
     if (initialized) {
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     } else {
-      return mediapipe::Status(mediapipe::StatusCode::kInternal,
-                               "Mfcc::Initialize returned uninitialized");
+      return absl::Status(absl::StatusCode::kInternal,
+                          "Mfcc::Initialize returned uninitialized");
     }
   }
 
@@ -228,13 +228,13 @@ REGISTER_CALCULATOR(MfccCalculator);
 // if you ask for too many channels.
 class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     return FramewiseTransformCalculatorBase::GetContract(cc);
   }
 
  private:
-  mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-                                       CalculatorContext* cc) override {
+  absl::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                  CalculatorContext* cc) override {
     MelSpectrumCalculatorOptions mel_spectrum_options =
         cc->Options<MelSpectrumCalculatorOptions>();
     mel_filterbank_.reset(new audio_dsp::MelFilterbank());
@@ -245,7 +245,7 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
     // audio_dsp::MelFilterBank needs to know this to
     // correctly interpret the spectrogram bins.
     if (!header.has_audio_sample_rate()) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
                        PortableDebugString(header)));
     }
@@ -255,10 +255,10 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
         mel_spectrum_options.max_frequency_hertz());
 
     if (initialized) {
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     } else {
-      return mediapipe::Status(mediapipe::StatusCode::kInternal,
-                               "mfcc::Initialize returned uninitialized");
+      return absl::Status(absl::StatusCode::kInternal,
+                          "mfcc::Initialize returned uninitialized");
     }
   }
 

mediapipe/calculators/audio/mfcc_mel_calculators_test.cc

@@ -84,7 +84,7 @@ class FramewiseTransformCalculatorTest
     num_samples_per_packet_ = GenerateRandomNonnegInputStream(kNumPackets);
   }
 
-  mediapipe::Status Run() { return this->RunGraph(); }
+  absl::Status Run() { return this->RunGraph(); }
 
   void CheckResults(int expected_num_channels) {
     const auto& output_header =

mediapipe/calculators/audio/rational_factor_resample_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019, 2021 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,22 +16,18 @@
 
 #include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
 
-#include "audio/dsp/resampler_rational_factor.h"
+#include "audio/dsp/resampler_q.h"
 
-using audio_dsp::DefaultResamplingKernel;
-using audio_dsp::RationalFactorResampler;
 using audio_dsp::Resampler;
 
 namespace mediapipe {
-mediapipe::Status RationalFactorResampleCalculator::Process(
-    CalculatorContext* cc) {
+absl::Status RationalFactorResampleCalculator::Process(CalculatorContext* cc) {
   return ProcessInternal(cc->Inputs().Index(0).Get<Matrix>(), false, cc);
 }
 
-mediapipe::Status RationalFactorResampleCalculator::Close(
-    CalculatorContext* cc) {
+absl::Status RationalFactorResampleCalculator::Close(CalculatorContext* cc) {
   if (initial_timestamp_ == Timestamp::Unstarted()) {
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
   Matrix empty_input_frame(num_channels_, 0);
   return ProcessInternal(empty_input_frame, true, cc);
@@ -40,11 +36,8 @@ mediapipe::Status RationalFactorResampleCalculator::Close(
 namespace {
 void CopyChannelToVector(const Matrix& matrix, int channel,
                          std::vector<float>* vec) {
-  vec->clear();
-  vec->reserve(matrix.cols());
-  for (int sample = 0; sample < matrix.cols(); ++sample) {
-    vec->push_back(matrix(channel, sample));
-  }
+  vec->resize(matrix.cols());
+  Eigen::Map<Eigen::ArrayXf>(vec->data(), vec->size()) = matrix.row(channel);
 }
 
 void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
@@ -53,17 +46,14 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
     matrix->resize(matrix->rows(), vec.size());
   } else {
     CHECK_EQ(vec.size(), matrix->cols());
-    CHECK_LT(channel, matrix->rows());
-  }
-  for (int sample = 0; sample < matrix->cols(); ++sample) {
-    (*matrix)(channel, sample) = vec[sample];
   }
+  CHECK_LT(channel, matrix->rows());
+  matrix->row(channel) =
+      Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
 }
-
 }  // namespace
 
-mediapipe::Status RationalFactorResampleCalculator::Open(
-    CalculatorContext* cc) {
+absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
   RationalFactorResampleCalculatorOptions resample_options =
       cc->Options<RationalFactorResampleCalculatorOptions>();
 
@@ -88,7 +78,7 @@ mediapipe::Status RationalFactorResampleCalculator::Open(
                                resample_options);
       if (!r) {
         LOG(ERROR) << "Failed to initialize resampler.";
-        return mediapipe::UnknownError("Failed to initialize resampler.");
+        return absl::UnknownError("Failed to initialize resampler.");
       }
     }
   }
@@ -106,10 +96,10 @@ mediapipe::Status RationalFactorResampleCalculator::Open(
   initial_timestamp_ = Timestamp::Unstarted();
   check_inconsistent_timestamps_ =
       resample_options.check_inconsistent_timestamps();
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
+absl::Status RationalFactorResampleCalculator::ProcessInternal(
     const Matrix& input_frame, bool should_flush, CalculatorContext* cc) {
   if (initial_timestamp_ == Timestamp::Unstarted()) {
     initial_timestamp_ = cc->InputTimestamp();
@@ -131,7 +121,7 @@ mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
     *output_frame = input_frame;
   } else {
     if (!Resample(input_frame, output_frame.get(), should_flush)) {
-      return mediapipe::UnknownError("Resample() failed.");
+      return absl::UnknownError("Resample() failed.");
     }
   }
   cumulative_output_samples_ += output_frame->cols();
@@ -139,7 +129,7 @@ mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
   if (output_frame->cols() > 0) {
     cc->Outputs().Index(0).Add(output_frame.release(), output_timestamp);
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 bool RationalFactorResampleCalculator::Resample(const Matrix& input_frame,
@@ -167,25 +157,28 @@ RationalFactorResampleCalculator::ResamplerFromOptions(
   std::unique_ptr<Resampler<float>> resampler;
   const auto& rational_factor_options =
       options.resampler_rational_factor_options();
-  std::unique_ptr<DefaultResamplingKernel> kernel;
+  audio_dsp::QResamplerParams params;
   if (rational_factor_options.has_radius() &&
       rational_factor_options.has_cutoff() &&
       rational_factor_options.has_kaiser_beta()) {
-    kernel = absl::make_unique<DefaultResamplingKernel>(
-        source_sample_rate, target_sample_rate,
-        rational_factor_options.radius(), rational_factor_options.cutoff(),
-        rational_factor_options.kaiser_beta());
-  } else {
-    kernel = absl::make_unique<DefaultResamplingKernel>(source_sample_rate,
-                                                        target_sample_rate);
+    // Convert RationalFactorResampler kernel parameters to QResampler
+    // settings.
+    params.filter_radius_factor =
+        rational_factor_options.radius() *
+        std::min(1.0, target_sample_rate / source_sample_rate);
+    params.cutoff_proportion = 2 * rational_factor_options.cutoff() /
+                               std::min(source_sample_rate, target_sample_rate);
+    params.kaiser_beta = rational_factor_options.kaiser_beta();
   }
-
   // Set large enough so that the resampling factor between common sample
   // rates (e.g. 8kHz, 16kHz, 22.05kHz, 32kHz, 44.1kHz, 48kHz) is exact, and
   // that any factor is represented with error less than 0.025%.
-  const int kMaxDenominator = 2000;
-  resampler = absl::make_unique<RationalFactorResampler<float>>(
-      *kernel, kMaxDenominator);
+  params.max_denominator = 2000;
+
+  // NOTE: QResampler supports multichannel resampling, so the code might be
+  // simplified using a single instance rather than one per channel.
+  resampler = absl::make_unique<audio_dsp::QResampler<float>>(
+      source_sample_rate, target_sample_rate, /*num_channels=*/1, params);
   if (resampler != nullptr && !resampler->Valid()) {
     resampler = std::unique_ptr<Resampler<float>>();
   }

mediapipe/calculators/audio/rational_factor_resample_calculator.h

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019, 2021 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -36,28 +36,31 @@ namespace mediapipe {
 // stream's sampling rate is specified by target_sample_rate in the
 // RationalFactorResampleCalculatorOptions.  The output time series may have
 // a varying number of samples per frame.
+//
+// NOTE: This calculator uses QResampler, despite the name, which supersedes
+// RationalFactorResampler.
 class RationalFactorResampleCalculator : public CalculatorBase {
  public:
   struct TestAccess;
 
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<Matrix>(
         // Single input stream with TimeSeriesHeader.
     );
     cc->Outputs().Index(0).Set<Matrix>(
         // Resampled stream with TimeSeriesHeader.
     );
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
   // Returns FAIL if the input stream header is invalid or if the
   // resampler cannot be initialized.
-  mediapipe::Status Open(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
   // Resamples a packet of TimeSeries data.  Returns FAIL if the
   // resampler state becomes inconsistent.
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
   // Flushes any remaining state.  Returns FAIL if the resampler state
   // becomes inconsistent.
-  mediapipe::Status Close(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
 
  protected:
   typedef audio_dsp::Resampler<float> ResamplerType;
@@ -72,8 +75,8 @@ class RationalFactorResampleCalculator : public CalculatorBase {
   // Does Timestamp bookkeeping and resampling common to Process() and
   // Close().  Returns FAIL if the resampler state becomes
   // inconsistent.
-  mediapipe::Status ProcessInternal(const Matrix& input_frame,
-                                    bool should_flush, CalculatorContext* cc);
+  absl::Status ProcessInternal(const Matrix& input_frame, bool should_flush,
+                               CalculatorContext* cc);
 
   // Uses the internal resampler_ objects to actually resample each
   // row of the input TimeSeries.  Returns false if the resampler

mediapipe/calculators/audio/rational_factor_resample_calculator.proto

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019, 2021 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,6 +18,8 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
+// NOTE: This calculator uses QResampler, despite the name, which supersedes
+// RationalFactorResampler.
 message RationalFactorResampleCalculatorOptions {
   extend CalculatorOptions {
     optional RationalFactorResampleCalculatorOptions ext = 259760074;
@@ -27,8 +29,7 @@ message RationalFactorResampleCalculatorOptions {
   // stream.  Required.  Must be greater than 0.
   optional double target_sample_rate = 1;
 
-  // Parameters for initializing the RationalFactorResampler. See
-  // RationalFactorResampler for more details.
+  // Parameters for initializing QResampler. See QResampler for more details.
   message ResamplerRationalFactorOptions {
     // Kernel radius in units of input samples.
     optional double radius = 1;

mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc

@@ -80,7 +80,7 @@ class RationalFactorResampleCalculatorTest
   }
 
   // Initializes and runs the test graph.
-  mediapipe::Status Run(double output_sample_rate) {
+  absl::Status Run(double output_sample_rate) {
     options_.set_target_sample_rate(output_sample_rate);
     InitializeGraph();
 
@@ -120,7 +120,6 @@ class RationalFactorResampleCalculatorTest
 
     // The exact number of expected samples may vary based on the implementation
     // of the resampler since the exact value is not an integer.
-    // TODO: Reduce this offset to + 1 once cl/185829520 is submitted.
     const double expected_num_output_samples = num_input_samples_ * factor;
     EXPECT_LE(ceil(expected_num_output_samples), num_output_samples);
     EXPECT_GE(ceil(expected_num_output_samples) + 11, num_output_samples);

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -66,7 +66,7 @@ namespace mediapipe {
 // analysis frame will advance from its predecessor by the same time step.
 class SpectrogramCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<Matrix>(
         // Input stream with TimeSeriesHeader.
     );
@@ -96,26 +96,34 @@ class SpectrogramCalculator : public CalculatorBase {
         );
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   // Returns FAIL if the input stream header is invalid.
-  mediapipe::Status Open(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
 
   // Outputs at most one packet consisting of a single Matrix with one or
   // more columns containing the spectral values from as many input frames
   // as are completed by the input samples.  Always returns OK.
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 
   // Performs zero-padding and processing of any remaining samples
   // if pad_final_packet is set.
   // Returns OK.
-  mediapipe::Status Close(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
 
  private:
   Timestamp CurrentOutputTimestamp(CalculatorContext* cc) {
     if (use_local_timestamp_) {
-      return cc->InputTimestamp();
+      const Timestamp now = cc->InputTimestamp();
+      if (now == Timestamp::Done()) {
+        // During Close the timestamp is not available, send an estimate.
+        return last_local_output_timestamp_ +
+               round(last_completed_frames_ * frame_step_samples() *
+                     Timestamp::kTimestampUnitsPerSecond / input_sample_rate_);
+      }
+      last_local_output_timestamp_ = now;
+      return now;
     }
     return CumulativeOutputTimestamp();
   }
@@ -138,17 +146,20 @@ class SpectrogramCalculator : public CalculatorBase {
   // Convert the output of the spectrogram object into a Matrix (or an
   // Eigen::MatrixXcf if complex-valued output is requested) and pass to
   // MediaPipe output.
-  mediapipe::Status ProcessVector(const Matrix& input_stream,
-                                  CalculatorContext* cc);
+  absl::Status ProcessVector(const Matrix& input_stream, CalculatorContext* cc);
 
   // Templated function to process either real- or complex-output spectrogram.
   template <class OutputMatrixType>
-  mediapipe::Status ProcessVectorToOutput(
+  absl::Status ProcessVectorToOutput(
       const Matrix& input_stream,
       const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
       CalculatorContext* cc);
 
+  // Use the MediaPipe timestamp instead of the estimated one. Useful when the
+  // data is intermittent.
   bool use_local_timestamp_;
+  Timestamp last_local_output_timestamp_;
+
   double input_sample_rate_;
   bool pad_final_packet_;
   int frame_duration_samples_;
@@ -157,6 +168,9 @@ class SpectrogramCalculator : public CalculatorBase {
   int64 cumulative_input_samples_;
   // How many frames we've emitted, used for calculating output time stamps.
   int64 cumulative_completed_frames_;
+  // How many frames were emitted last, used for estimating the timestamp on
+  // Close when use_local_timestamp_ is true;
+  int64 last_completed_frames_;
   Timestamp initial_input_timestamp_;
   int num_input_channels_;
   // How many frequency bins we emit (=N_FFT/2 + 1).
@@ -177,7 +191,7 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 // Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0).
 const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
-mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
+absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
   SpectrogramCalculatorOptions spectrogram_options =
       cc->Options<SpectrogramCalculatorOptions>();
 
@@ -271,11 +285,20 @@ mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
         Adopt(multichannel_output_header.release()));
   }
   cumulative_completed_frames_ = 0;
+  last_completed_frames_ = 0;
   initial_input_timestamp_ = Timestamp::Unstarted();
-  return mediapipe::OkStatus();
+  if (use_local_timestamp_) {
+    // Inform the framework that the calculator will output packets at the same
+    // timestamps as input packets to enable packet queueing optimizations. The
+    // final packet (emitted from Close()) does not follow this rule but it's
+    // sufficient that its timestamp is strictly greater than the timestamp of
+    // the previous packet.
+    cc->SetOffset(0);
+  }
+  return absl::OkStatus();
 }
 
-mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
+absl::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
   if (initial_input_timestamp_ == Timestamp::Unstarted()) {
     initial_input_timestamp_ = cc->InputTimestamp();
   }
@@ -291,7 +314,7 @@ mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
 }
 
 template <class OutputMatrixType>
-mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
+absl::Status SpectrogramCalculator::ProcessVectorToOutput(
     const Matrix& input_stream,
     const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
     CalculatorContext* cc) {
@@ -311,8 +334,8 @@ mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
 
     if (!spectrogram_generators_[channel]->ComputeSpectrogram(
             input_vector, &output_vectors)) {
-      return mediapipe::Status(mediapipe::StatusCode::kInternal,
-                               "Spectrogram returned failure");
+      return absl::Status(absl::StatusCode::kInternal,
+                          "Spectrogram returned failure");
     }
     if (channel == 0) {
       // Record the number of time frames we expect from each channel.
@@ -354,12 +377,19 @@ mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
           CurrentOutputTimestamp(cc));
     }
     cumulative_completed_frames_ += output_vectors.size();
+    last_completed_frames_ = output_vectors.size();
+    if (!use_local_timestamp_) {
+      // In non-local timestamp mode the timestamp of the next packet will be
+      // equal to CumulativeOutputTimestamp(). Inform the framework about this
+      // fact to enable packet queueing optimizations.
+      cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
+    }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status SpectrogramCalculator::ProcessVector(
-    const Matrix& input_stream, CalculatorContext* cc) {
+absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
+                                                  CalculatorContext* cc) {
   switch (output_type_) {
     // These blocks deliberately ignore clang-format to preserve the
     // "silhouette" of the different cases.
@@ -394,13 +424,13 @@ mediapipe::Status SpectrogramCalculator::ProcessVector(
     }
     // clang-format on
     default: {
-      return mediapipe::Status(mediapipe::StatusCode::kInvalidArgument,
-                               "Unrecognized spectrogram output type.");
+      return absl::Status(absl::StatusCode::kInvalidArgument,
+                          "Unrecognized spectrogram output type.");
     }
   }
 }
 
-mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
+absl::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
   if (cumulative_input_samples_ > 0 && pad_final_packet_) {
     // We can flush any remaining samples by sending frame_step_samples - 1
     // zeros to the Process method, and letting it do its thing,
@@ -416,7 +446,7 @@ mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
         Matrix::Zero(num_input_channels_, required_padding_samples), cc);
   }
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/calculators/audio/spectrogram_calculator_test.cc

@@ -50,7 +50,7 @@ class SpectrogramCalculatorTest
   }
 
   // Initializes and runs the test graph.
-  mediapipe::Status Run() {
+  absl::Status Run() {
     // Now that options are set, we can set up some internal constants.
     frame_duration_samples_ =
         round(options_.frame_duration_seconds() * input_sample_rate_);

mediapipe/calculators/audio/stabilized_log_calculator.cc

@@ -41,17 +41,17 @@ namespace mediapipe {
 // }
 class StabilizedLogCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<Matrix>(
         // Input stream with TimeSeriesHeader.
     );
     cc->Outputs().Index(0).Set<Matrix>(
         // Output stabilized log stream with TimeSeriesHeader.
     );
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     StabilizedLogCalculatorOptions stabilized_log_calculator_options =
         cc->Options<StabilizedLogCalculatorOptions>();
 
@@ -70,23 +70,23 @@ class StabilizedLogCalculator : public CalculatorBase {
       cc->Outputs().Index(0).SetHeader(
           Adopt(new TimeSeriesHeader(input_header)));
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
     if (input_matrix.array().isNaN().any()) {
-      return mediapipe::InvalidArgumentError("NaN input to log operation.");
+      return absl::InvalidArgumentError("NaN input to log operation.");
     }
     if (check_nonnegativity_) {
       if (input_matrix.minCoeff() < 0.0) {
-        return mediapipe::OutOfRangeError("Negative input to log operation.");
+        return absl::OutOfRangeError("Negative input to log operation.");
       }
     }
     std::unique_ptr<Matrix> output_frame(new Matrix(
         output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
     cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -66,26 +66,26 @@ namespace mediapipe {
 // cumulative_completed_samples / sample_rate_.
 class TimeSeriesFramerCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<Matrix>(
         // Input stream with TimeSeriesHeader.
     );
     cc->Outputs().Index(0).Set<Matrix>(
         // Fixed length time series Packets with TimeSeriesHeader.
     );
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   // Returns FAIL if the input stream header is invalid.
-  mediapipe::Status Open(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
 
   // Outputs as many framed packets as possible given the accumulated
   // input.  Always returns OK.
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 
   // Flushes any remaining samples in a zero-padded packet.  Always
   // returns OK.
-  mediapipe::Status Close(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
 
  private:
   // Adds input data to the internal buffer.
@@ -134,7 +134,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   // emulate_fractional_frame_overlap is true.
   double average_frame_step_samples_;
   int samples_still_to_drop_;
-  int64 cumulative_input_samples_;
   int64 cumulative_output_frames_;
   // "Completed" samples are samples that are no longer needed because
   // the framer has completely stepped past them (taking into account
@@ -163,8 +162,6 @@ void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
     sample_buffer_.emplace_back(std::make_pair(
         input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
   }
-
-  cumulative_input_samples_ += input_frame.cols();
 }
 
 void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
@@ -203,9 +200,15 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
     ++cumulative_output_frames_;
     cumulative_completed_samples_ += frame_step_samples;
   }
+  if (!use_local_timestamp_) {
+    // In non-local timestamp mode the timestamp of the next packet will be
+    // equal to CumulativeOutputTimestamp(). Inform the framework about this
+    // fact to enable packet queueing optimizations.
+    cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
+  }
 }
 
-mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
   if (initial_input_timestamp_ == Timestamp::Unstarted()) {
     initial_input_timestamp_ = cc->InputTimestamp();
     current_timestamp_ = initial_input_timestamp_;
@@ -214,10 +217,10 @@ mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
   EnqueueInput(cc);
   FrameOutput(cc);
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
+absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
   while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
     sample_buffer_.pop_front();
     --samples_still_to_drop_;
@@ -234,10 +237,10 @@ mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
                                CurrentOutputTimestamp());
   }
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
+absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   TimeSeriesFramerCalculatorOptions framer_options =
       cc->Options<TimeSeriesFramerCalculatorOptions>();
 
@@ -286,7 +289,6 @@ mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   }
   cc->Outputs().Index(0).SetHeader(Adopt(output_header));
   cumulative_completed_samples_ = 0;
-  cumulative_input_samples_ = 0;
   cumulative_output_frames_ = 0;
   samples_still_to_drop_ = 0;
   initial_input_timestamp_ = Timestamp::Unstarted();
@@ -317,7 +319,7 @@ mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   }
   use_local_timestamp_ = framer_options.use_local_timestamp();
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/calculators/audio/time_series_framer_calculator_test.cc

@@ -69,7 +69,7 @@ class TimeSeriesFramerCalculatorTest
   }
 
   // Initializes and runs the test graph.
-  mediapipe::Status Run() {
+  absl::Status Run() {
     InitializeGraph();
 
     FillInputHeader();
@@ -441,7 +441,7 @@ class TimeSeriesFramerCalculatorTimestampingTest
     }
   }
 
-  mediapipe::Status RunTimestampTest() {
+  absl::Status RunTimestampTest() {
     InitializeGraph();
     InitializeInputForTimeStampingTest();
     FillInputHeader();

mediapipe/calculators/core/BUILD

@@ -146,6 +146,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:status",
     ],
@@ -248,6 +249,8 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:tensor",
@@ -286,6 +289,7 @@ cc_library(
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -393,6 +397,7 @@ cc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
@@ -406,10 +411,10 @@ cc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -422,10 +427,10 @@ cc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -438,12 +443,27 @@ cc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/stream_handler:mux_input_stream_handler",
     ],
     alwayslink = 1,
 )
 
+cc_library(
+    name = "nonzero_calculator",
+    srcs = ["nonzero_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "mux_calculator_test",
     srcs = ["mux_calculator_test.cc"],
@@ -550,6 +570,7 @@ cc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:ret_check",
     ],
     alwayslink = 1,
@@ -606,6 +627,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
@@ -768,7 +790,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -785,7 +807,7 @@ cc_test(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:validate_type",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
 )
 
@@ -958,6 +980,7 @@ cc_library(
     deps = [
         ":sequence_shift_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
@@ -1007,6 +1030,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:logging",
@@ -1014,7 +1038,7 @@ cc_library(
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/memory",
-        "@eigen_archive//:eigen",
+        "@eigen_archive//:eigen3",
     ],
     alwayslink = 1,
 )
@@ -1042,6 +1066,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ],

mediapipe/calculators/core/add_header_calculator.cc

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/logging.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // Attach the header from a stream or side input to another stream.
 //
@@ -42,49 +44,41 @@ namespace mediapipe {
 //   output_stream: "audio_with_header"
 // }
 //
-class AddHeaderCalculator : public CalculatorBase {
+class AddHeaderCalculator : public Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    bool has_side_input = false;
-    bool has_header_stream = false;
-    if (cc->InputSidePackets().HasTag("HEADER")) {
-      cc->InputSidePackets().Tag("HEADER").SetAny();
-      has_side_input = true;
-    }
-    if (cc->Inputs().HasTag("HEADER")) {
-      cc->Inputs().Tag("HEADER").SetNone();
-      has_header_stream = true;
-    }
-    if (has_side_input == has_header_stream) {
-      return mediapipe::InvalidArgumentError(
+  static constexpr Input<NoneType>::Optional kHeader{"HEADER"};
+  static constexpr SideInput<AnyType>::Optional kHeaderSide{"HEADER"};
+  static constexpr Input<AnyType> kData{"DATA"};
+  static constexpr Output<SameType<kData>> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kHeader, kHeaderSide, kData, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    if (kHeader(cc).IsConnected() == kHeaderSide(cc).IsConnected()) {
+      return absl::InvalidArgumentError(
           "Header must be provided via exactly one of side input and input "
           "stream");
     }
-    cc->Inputs().Tag("DATA").SetAny();
-    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
-    Packet header;
-    if (cc->InputSidePackets().HasTag("HEADER")) {
-      header = cc->InputSidePackets().Tag("HEADER");
-    }
-    if (cc->Inputs().HasTag("HEADER")) {
-      header = cc->Inputs().Tag("HEADER").Header();
-    }
+  absl::Status Open(CalculatorContext* cc) override {
+    const PacketBase& header =
+        kHeader(cc).IsConnected() ? kHeader(cc).Header() : kHeaderSide(cc);
     if (!header.IsEmpty()) {
-      cc->Outputs().Index(0).SetHeader(header);
+      kOut(cc).SetHeader(header);
     }
-    cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    cc->SetOffset(0);
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
-    cc->Outputs().Index(0).AddPacket(cc->Inputs().Tag("DATA").Value());
-    return mediapipe::OkStatus();
+  absl::Status Process(CalculatorContext* cc) override {
+    kOut(cc).Send(kData(cc).packet());
+    return absl::OkStatus();
   }
 };
 
-REGISTER_CALCULATOR(AddHeaderCalculator);
+MEDIAPIPE_REGISTER_NODE(AddHeaderCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -153,7 +153,7 @@ TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
   }
 
   // Run should fail because header can only be provided one way.
-  EXPECT_EQ(runner.Run().code(), mediapipe::InvalidArgumentError("").code());
+  EXPECT_EQ(runner.Run().code(), absl::InvalidArgumentError("").code());
 }
 
 }  // namespace mediapipe

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -42,22 +42,22 @@ REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
 
 class IncrementCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<int>();
     cc->Outputs().Index(0).Set<int>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     const int& input_int = cc->Inputs().Index(0).Get<int>();
     auto output_int = absl::make_unique<int>(input_int + 1);
     cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 
@@ -166,19 +166,19 @@ TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
 // bound update.
 class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->SetProcessTimestampBounds(true);
     cc->Inputs().Index(0).Set<std::vector<int>>();
     cc->Outputs().Index(0).Set<std::vector<int>>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (!cc->Inputs().Index(0).IsEmpty()) {
       cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
     } else {
@@ -186,7 +186,7 @@ class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
           MakePacket<std::vector<int>>(std::vector<int>())
               .At(cc->InputTimestamp()));
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 
@@ -311,24 +311,24 @@ TEST_F(BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest, MultipleVectors) {
 
 class MultiplierCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<int>();
     cc->Inputs().Index(1).Set<int>();
     cc->Outputs().Index(0).Set<int>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     const int& input_int = cc->Inputs().Index(0).Get<int>();
     const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
     auto output_int = absl::make_unique<int>(input_int * multiplier_int);
     cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 

mediapipe/calculators/core/begin_loop_calculator.h

@@ -61,7 +61,7 @@ class BeginLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;
 
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     // The below enables processing of timestamp bound updates, and that enables
     // correct timestamp propagation by the companion EndLoopCalculator.
     //
@@ -106,10 +106,10 @@ class BeginLoopCalculator : public CalculatorBase {
       }
     }
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     Timestamp last_timestamp = loop_internal_timestamp_;
     if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
       const IterableT& collection =
@@ -139,7 +139,7 @@ class BeginLoopCalculator : public CalculatorBase {
         .AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
                        .At(Timestamp(loop_internal_timestamp_ - 1)));
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/clip_vector_size_calculator.h

@@ -43,13 +43,13 @@ namespace mediapipe {
 template <typename T>
 class ClipVectorSizeCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     RET_CHECK(cc->Inputs().NumEntries() == 1);
     RET_CHECK(cc->Outputs().NumEntries() == 1);
 
     if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
             .max_vec_size() < 1) {
-      return mediapipe::InternalError(
+      return absl::InternalError(
           "max_vec_size should be greater than or equal to 1.");
     }
 
@@ -60,10 +60,10 @@ class ClipVectorSizeCalculator : public CalculatorBase {
       cc->InputSidePackets().Index(0).Set<int>();
     }
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
     max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
                         .max_vec_size();
@@ -72,23 +72,23 @@ class ClipVectorSizeCalculator : public CalculatorBase {
         !cc->InputSidePackets().Index(0).IsEmpty()) {
       max_vec_size_ = cc->InputSidePackets().Index(0).Get<int>();
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (max_vec_size_ < 1) {
-      return mediapipe::InternalError(
+      return absl::InternalError(
           "max_vec_size should be greater than or equal to 1.");
     }
     if (cc->Inputs().Index(0).IsEmpty()) {
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     }
 
     return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
   }
 
   template <typename U>
-  mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
+  absl::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
     auto output = absl::make_unique<std::vector<U>>();
     const std::vector<U>& input_vector =
         cc->Inputs().Index(0).Get<std::vector<U>>();
@@ -100,24 +100,23 @@ class ClipVectorSizeCalculator : public CalculatorBase {
       }
     }
     cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   template <typename U>
-  mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
+  absl::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
     return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
   }
 
   template <typename U>
-  mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
-                                             CalculatorContext* cc) {
+  absl::Status ConsumeAndClipVectorSize(std::true_type, CalculatorContext* cc) {
     auto output = absl::make_unique<std::vector<U>>();
-    mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+    absl::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
         cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
 
     if (input_status.ok()) {
       std::unique_ptr<std::vector<U>> input_vector =
-          std::move(input_status).ValueOrDie();
+          std::move(input_status).value();
       auto begin_it = input_vector->begin();
       auto end_it = input_vector->end();
       if (max_vec_size_ < input_vector->size()) {
@@ -129,13 +128,13 @@ class ClipVectorSizeCalculator : public CalculatorBase {
       return input_status.status();
     }
     cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   template <typename U>
-  mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
-                                             CalculatorContext* cc) {
-    return mediapipe::InternalError(
+  absl::Status ConsumeAndClipVectorSize(std::false_type,
+                                        CalculatorContext* cc) {
+    return absl::InternalError(
         "Cannot copy or move input vectors and clip their size.");
   }
 

mediapipe/calculators/core/concatenate_detection_vector_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019-2020 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,14 +20,16 @@
 namespace mediapipe {
 
 // Example config:
+//
 // node {
 //   calculator: "ConcatenateDetectionVectorCalculator"
 //   input_stream: "detection_vector_1"
 //   input_stream: "detection_vector_2"
 //   output_stream: "concatenated_detection_vector"
 // }
+//
 typedef ConcatenateVectorCalculator<::mediapipe::Detection>
     ConcatenateDetectionVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateDetectionVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateDetectionVectorCalculator);
 
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc

@@ -16,6 +16,7 @@
 #define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_  // NOLINT
 
 #include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
@@ -23,61 +24,55 @@
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // Concatenates several NormalizedLandmarkList protos following stream index
 // order. This class assumes that every input stream contains a
 // NormalizedLandmarkList proto object.
-class ConcatenateNormalizedLandmarkListCalculator : public CalculatorBase {
+class ConcatenateNormalizedLandmarkListCalculator : public Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().NumEntries() != 0);
-    RET_CHECK(cc->Outputs().NumEntries() == 1);
+  static constexpr Input<NormalizedLandmarkList>::Multiple kIn{""};
+  static constexpr Output<NormalizedLandmarkList> kOut{""};
 
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      cc->Inputs().Index(i).Set<NormalizedLandmarkList>();
-    }
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
 
-    cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
-
-    return mediapipe::OkStatus();
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_GE(kIn(cc).Count(), 1);
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
+  absl::Status Open(CalculatorContext* cc) override {
     only_emit_if_all_present_ =
         cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
             .only_emit_if_all_present();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (only_emit_if_all_present_) {
-      for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-        if (cc->Inputs().Index(i).IsEmpty()) return mediapipe::OkStatus();
+      for (const auto& input : kIn(cc)) {
+        if (input.IsEmpty()) return absl::OkStatus();
       }
     }
 
     NormalizedLandmarkList output;
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      if (cc->Inputs().Index(i).IsEmpty()) continue;
-      const NormalizedLandmarkList& input =
-          cc->Inputs().Index(i).Get<NormalizedLandmarkList>();
-      for (int j = 0; j < input.landmark_size(); ++j) {
-        const NormalizedLandmark& input_landmark = input.landmark(j);
-        *output.add_landmark() = input_landmark;
+    for (const auto& input : kIn(cc)) {
+      if (input.IsEmpty()) continue;
+      const NormalizedLandmarkList& list = *input;
+      for (int j = 0; j < list.landmark_size(); ++j) {
+        *output.add_landmark() = list.landmark(j);
       }
     }
-    cc->Outputs().Index(0).AddPacket(
-        MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
-    return mediapipe::OkStatus();
+    kOut(cc).Send(std::move(output));
+    return absl::OkStatus();
   }
 
  private:
   bool only_emit_if_all_present_;
 };
+MEDIAPIPE_REGISTER_NODE(ConcatenateNormalizedLandmarkListCalculator);
 
-REGISTER_CALCULATOR(ConcatenateNormalizedLandmarkListCalculator);
-
+}  // namespace api2
 }  // namespace mediapipe
 
 // NOLINTNEXTLINE

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -25,7 +25,7 @@
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
-#endif  //  !MEDIAPIPE_DISABLE_GPU
+#endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 
 namespace mediapipe {
 
@@ -37,7 +37,7 @@ namespace mediapipe {
 //   output_stream: "concatenated_float_vector"
 // }
 typedef ConcatenateVectorCalculator<float> ConcatenateFloatVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateFloatVectorCalculator);
 
 // Example config:
 // node {
@@ -47,13 +47,13 @@ REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
 //   output_stream: "concatenated_int32_vector"
 // }
 typedef ConcatenateVectorCalculator<int32> ConcatenateInt32VectorCalculator;
-REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateInt32VectorCalculator);
 
 typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
-REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
 
 typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
 
 // Example config:
 // node {
@@ -64,31 +64,31 @@ REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
 // }
 typedef ConcatenateVectorCalculator<TfLiteTensor>
     ConcatenateTfLiteTensorVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateTfLiteTensorVectorCalculator);
 
 typedef ConcatenateVectorCalculator<Tensor> ConcatenateTensorVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateTensorVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateTensorVectorCalculator);
 
 typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
     ConcatenateLandmarkVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkVectorCalculator);
 
 typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmarkList>
     ConcatenateLandmarListVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateLandmarListVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarListVectorCalculator);
 
 typedef ConcatenateVectorCalculator<mediapipe::ClassificationList>
     ConcatenateClassificationListVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateClassificationListVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListVectorCalculator);
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
     ConcatenateGlBufferVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateGlBufferVectorCalculator);
 #endif
 
 typedef ConcatenateVectorCalculator<mediapipe::RenderData>
     ConcatenateRenderDataVectorCalculator;
-REGISTER_CALCULATOR(ConcatenateRenderDataVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
 
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.h

@@ -20,120 +20,96 @@
 #include <vector>
 
 #include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+// Note: since this is a calculator template that can be included by other
+// source files, we do not place this in namespace api2 directly, but qualify
+// the api2 names below, to avoid changing the visible name of the class.
+// We cannot simply write "using mediapipe::api2" since it's a header file.
+// This distinction will go away once api2 is finalized.
 
 // Concatenates several objects of type T or std::vector<T> following stream
 // index order. This class assumes that every input stream contains either T or
 // vector<T> type. To use this class for a particular type T, regisiter a
 // calculator using ConcatenateVectorCalculator<T>.
 template <typename T>
-class ConcatenateVectorCalculator : public CalculatorBase {
+class ConcatenateVectorCalculator : public api2::Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().NumEntries() != 0);
-    RET_CHECK(cc->Outputs().NumEntries() == 1);
+  static constexpr
+      typename api2::Input<api2::OneOf<T, std::vector<T>>>::Multiple kIn{""};
+  static constexpr api2::Output<std::vector<T>> kOut{""};
 
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      // Actual type T or vector<T> will be validated in Process().
-      cc->Inputs().Index(i).SetAny();
-    }
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
 
-    cc->Outputs().Index(0).Set<std::vector<T>>();
-
-    return mediapipe::OkStatus();
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_GE(kIn(cc).Count(), 1);
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
+  absl::Status Open(CalculatorContext* cc) override {
     only_emit_if_all_present_ =
         cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
             .only_emit_if_all_present();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (only_emit_if_all_present_) {
-      for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-        if (cc->Inputs().Index(i).IsEmpty()) return mediapipe::OkStatus();
+      for (const auto& input : kIn(cc)) {
+        if (input.IsEmpty()) return ::absl::OkStatus();
       }
     }
-
     return ConcatenateVectors<T>(std::is_copy_constructible<T>(), cc);
   }
 
   template <typename U>
-  mediapipe::Status ConcatenateVectors(std::true_type, CalculatorContext* cc) {
-    auto output = absl::make_unique<std::vector<U>>();
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      auto& input = cc->Inputs().Index(i);
-
+  absl::Status ConcatenateVectors(std::true_type, CalculatorContext* cc) {
+    auto output = std::vector<U>();
+    for (const auto& input : kIn(cc)) {
       if (input.IsEmpty()) continue;
-
-      if (input.Value().ValidateAsType<U>().ok()) {
-        const U& value = input.Get<U>();
-        output->push_back(value);
-      } else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
-        const std::vector<U>& value = input.Get<std::vector<U>>();
-        output->insert(output->end(), value.begin(), value.end());
-      } else {
-        return mediapipe::InvalidArgumentError("Invalid input stream type.");
-      }
+      input.Visit([&output](const U& value) { output.push_back(value); },
+                  [&output](const std::vector<U>& value) {
+                    output.insert(output.end(), value.begin(), value.end());
+                  });
     }
-    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    kOut(cc).Send(std::move(output));
+    return absl::OkStatus();
   }
 
   template <typename U>
-  mediapipe::Status ConcatenateVectors(std::false_type, CalculatorContext* cc) {
+  absl::Status ConcatenateVectors(std::false_type, CalculatorContext* cc) {
     return ConsumeAndConcatenateVectors<T>(std::is_move_constructible<U>(), cc);
   }
 
   template <typename U>
-  mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
-                                                 CalculatorContext* cc) {
-    auto output = absl::make_unique<std::vector<U>>();
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      auto& input = cc->Inputs().Index(i);
-
+  absl::Status ConsumeAndConcatenateVectors(std::true_type,
+                                            CalculatorContext* cc) {
+    auto output = std::vector<U>();
+    for (auto input : kIn(cc)) {
       if (input.IsEmpty()) continue;
-
-      if (input.Value().ValidateAsType<U>().ok()) {
-        mediapipe::StatusOr<std::unique_ptr<U>> value_status =
-            input.Value().Consume<U>();
-        if (value_status.ok()) {
-          std::unique_ptr<U> value = std::move(value_status).ValueOrDie();
-          output->push_back(std::move(*value));
-        } else {
-          return value_status.status();
-        }
-      } else if (input.Value().ValidateAsType<std::vector<U>>().ok()) {
-        mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> value_status =
-            input.Value().Consume<std::vector<U>>();
-        if (value_status.ok()) {
-          std::unique_ptr<std::vector<U>> value =
-              std::move(value_status).ValueOrDie();
-          output->insert(output->end(), std::make_move_iterator(value->begin()),
-                         std::make_move_iterator(value->end()));
-        } else {
-          return value_status.status();
-        }
-      } else {
-        return mediapipe::InvalidArgumentError("Invalid input stream type.");
-      }
+      MP_RETURN_IF_ERROR(input.ConsumeAndVisit(
+          [&output](std::unique_ptr<U> value) {
+            output.push_back(std::move(*value));
+          },
+          [&output](std::unique_ptr<std::vector<U>> value) {
+            output.insert(output.end(), std::make_move_iterator(value->begin()),
+                          std::make_move_iterator(value->end()));
+          }));
     }
-    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-    return mediapipe::OkStatus();
+    kOut(cc).Send(std::move(output));
+    return absl::OkStatus();
   }
 
   template <typename U>
-  mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
-                                                 CalculatorContext* cc) {
-    return mediapipe::InternalError(
+  absl::Status ConsumeAndConcatenateVectors(std::false_type,
+                                            CalculatorContext* cc) {
+    return absl::InternalError(
         "Cannot copy or move inputs to concatenate them");
   }
 

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -28,7 +28,7 @@
 namespace mediapipe {
 
 typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
-REGISTER_CALCULATOR(TestConcatenateIntVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
 
 void AddInputVector(int index, const std::vector<int>& input, int64 timestamp,
                     CalculatorRunner* runner) {
@@ -384,7 +384,7 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
 
 typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
     TestConcatenateUniqueIntPtrCalculator;
-REGISTER_CALCULATOR(TestConcatenateUniqueIntPtrCalculator);
+MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
 
 TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
   /* Note: We don't use CalculatorRunner for this test because it keeps copies

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -54,7 +54,7 @@ namespace {}  // namespace
 // }
 class ConstantSidePacketCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     const auto& options =
         cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
@@ -80,14 +80,14 @@ class ConstantSidePacketCalculator : public CalculatorBase {
       } else if (packet_options.has_classification_list_value()) {
         packet.Set<ClassificationList>();
       } else {
-        return mediapipe::InvalidArgumentError(
+        return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     const auto& options =
         cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     int index = 0;
@@ -109,15 +109,15 @@ class ConstantSidePacketCalculator : public CalculatorBase {
         packet.Set(MakePacket<ClassificationList>(
             packet_options.classification_list_value()));
       } else {
-        return mediapipe::InvalidArgumentError(
+        return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
-    return mediapipe::OkStatus();
+  absl::Status Process(CalculatorContext* cc) override {
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -49,7 +49,7 @@ void DoTestSingleSidePacket(absl::string_view packet_spec,
 
   MP_ASSERT_OK(graph.GetOutputSidePacket("packet"));
   auto actual_value =
-      graph.GetOutputSidePacket("packet").ValueOrDie().template Get<T>();
+      graph.GetOutputSidePacket("packet").value().template Get<T>();
   EXPECT_EQ(actual_value, expected_value);
 }
 
@@ -89,28 +89,24 @@ TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
   MP_ASSERT_OK(graph.WaitUntilIdle());
 
   MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
-            256);
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").value().Get<int>(), 256);
   MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").value().Get<float>(),
             0.5f);
   MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
-  EXPECT_FALSE(
-      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  EXPECT_FALSE(graph.GetOutputSidePacket("bool_packet").value().Get<bool>());
   MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
-                .ValueOrDie()
-                .Get<std::string>(),
-            "string");
+  EXPECT_EQ(
+      graph.GetOutputSidePacket("string_packet").value().Get<std::string>(),
+      "string");
   MP_ASSERT_OK(graph.GetOutputSidePacket("another_string_packet"));
   EXPECT_EQ(graph.GetOutputSidePacket("another_string_packet")
-                .ValueOrDie()
+                .value()
                 .Get<std::string>(),
             "another string");
   MP_ASSERT_OK(graph.GetOutputSidePacket("another_int_packet"));
-  EXPECT_EQ(
-      graph.GetOutputSidePacket("another_int_packet").ValueOrDie().Get<int>(),
-      128);
+  EXPECT_EQ(graph.GetOutputSidePacket("another_int_packet").value().Get<int>(),
+            128);
 }
 
 TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
@@ -142,19 +138,16 @@ TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) {
   MP_ASSERT_OK(graph.WaitUntilIdle());
 
   MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").ValueOrDie().Get<int>(),
-            256);
+  EXPECT_EQ(graph.GetOutputSidePacket("int_packet").value().Get<int>(), 256);
   MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").ValueOrDie().Get<float>(),
+  EXPECT_EQ(graph.GetOutputSidePacket("float_packet").value().Get<float>(),
             0.5f);
   MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet"));
-  EXPECT_FALSE(
-      graph.GetOutputSidePacket("bool_packet").ValueOrDie().Get<bool>());
+  EXPECT_FALSE(graph.GetOutputSidePacket("bool_packet").value().Get<bool>());
   MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet"));
-  EXPECT_EQ(graph.GetOutputSidePacket("string_packet")
-                .ValueOrDie()
-                .Get<std::string>(),
-            "string");
+  EXPECT_EQ(
+      graph.GetOutputSidePacket("string_packet").value().Get<std::string>(),
+      "string");
 }
 
 TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) {

mediapipe/calculators/core/counting_source_calculator.cc

@@ -30,7 +30,7 @@ namespace mediapipe {
 // provided, then batches are of size 1.
 class CountingSourceCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Outputs().Index(0).Set<int>();
 
     if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN")) {
@@ -55,13 +55,13 @@ class CountingSourceCalculator : public CalculatorBase {
     if (cc->InputSidePackets().HasTag("INCREMENT")) {
       cc->InputSidePackets().Tag("INCREMENT").Set<int>();
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN") &&
         cc->InputSidePackets().Tag("ERROR_ON_OPEN").Get<bool>()) {
-      return mediapipe::NotFoundError("expected error");
+      return absl::NotFoundError("expected error");
     }
     if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
       error_count_ = cc->InputSidePackets().Tag("ERROR_COUNT").Get<int>();
@@ -83,12 +83,12 @@ class CountingSourceCalculator : public CalculatorBase {
       RET_CHECK_LT(0, increment_);
     }
     RET_CHECK(error_count_ >= 0 || max_count_ >= 0);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (error_count_ >= 0 && batch_counter_ >= error_count_) {
-      return mediapipe::InternalError("expected error");
+      return absl::InternalError("expected error");
     }
     if (max_count_ >= 0 && batch_counter_ >= max_count_) {
       return tool::StatusStop();
@@ -98,7 +98,7 @@ class CountingSourceCalculator : public CalculatorBase {
       counter_ += increment_;
     }
     ++batch_counter_;
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/dequantize_byte_array_calculator.cc

@@ -37,34 +37,34 @@ namespace mediapipe {
 
 class DequantizeByteArrayCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag("ENCODED").Set<std::string>();
     cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     const auto options =
         cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
     if (!options.has_max_quantized_value() ||
         !options.has_min_quantized_value()) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           "Both max_quantized_value and min_quantized_value must be provided "
           "in DequantizeByteArrayCalculatorOptions.");
     }
     float max_quantized_value = options.max_quantized_value();
     float min_quantized_value = options.min_quantized_value();
     if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           "max_quantized_value must be greater than min_quantized_value.");
     }
     float range = max_quantized_value - min_quantized_value;
     scalar_ = range / 255.0;
     bias_ = (range / 512.0) + min_quantized_value;
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     const std::string& encoded =
         cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
     std::vector<float> float_vector;
@@ -77,7 +77,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
         .Tag("FLOAT_VECTOR")
         .AddPacket(MakePacket<std::vector<float>>(float_vector)
                        .At(cc->InputTimestamp()));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/end_loop_calculator.h

@@ -57,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;
 
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
         << "Missing BATCH_END tagged input_stream.";
     cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
@@ -67,10 +67,10 @@ class EndLoopCalculator : public CalculatorBase {
 
     RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
     cc->Outputs().Tag("ITERABLE").Set<IterableT>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Process(CalculatorContext* cc) override {
     if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
       if (!input_stream_collection_) {
         input_stream_collection_.reset(new IterableT);
@@ -94,7 +94,7 @@ class EndLoopCalculator : public CalculatorBase {
             .SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -57,7 +57,7 @@ namespace mediapipe {
 //
 // The "ALLOW" stream indicates the transition between accepting frames and
 // dropping frames.  "ALLOW = true" indicates the start of accepting frames
-// including the current timestamp, and "ALLOW = true" indicates the start of
+// including the current timestamp, and "ALLOW = false" indicates the start of
 // dropping frames including the current timestamp.
 //
 // FlowLimiterCalculator provides limited support for multiple input streams.
@@ -67,7 +67,7 @@ namespace mediapipe {
 //
 class FlowLimiterCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     auto& side_inputs = cc->InputSidePackets();
     side_inputs.Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
     cc->Inputs().Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
@@ -81,10 +81,10 @@ class FlowLimiterCalculator : public CalculatorBase {
     cc->Outputs().Tag("ALLOW").Set<bool>().Optional();
     cc->SetInputStreamHandler("ImmediateInputStreamHandler");
     cc->SetProcessTimestampBounds(true);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     options_ = cc->Options<FlowLimiterCalculatorOptions>();
     options_ = tool::RetrieveOptions(options_, cc->InputSidePackets());
     if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
@@ -93,7 +93,7 @@ class FlowLimiterCalculator : public CalculatorBase {
     }
     input_queues_.resize(cc->Inputs().NumEntries(""));
     RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   // Returns true if an additional frame can be released for processing.
@@ -151,7 +151,7 @@ class FlowLimiterCalculator : public CalculatorBase {
   }
 
   // Releases input packets allowed by the max_in_flight constraint.
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     options_ = tool::RetrieveOptions(options_, cc->Inputs());
 
     // Process the FINISHED input stream.
@@ -216,7 +216,7 @@ class FlowLimiterCalculator : public CalculatorBase {
     }
 
     ProcessAuxiliaryInputs(cc);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -71,19 +71,19 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
 }
 
 // A Calculator::Process callback function.
-typedef std::function<mediapipe::Status(const InputStreamShardSet&,
-                                        OutputStreamShardSet*)>
+typedef std::function<absl::Status(const InputStreamShardSet&,
+                                   OutputStreamShardSet*)>
     ProcessFunction;
 
 // A testing callback function that passes through all packets.
-mediapipe::Status PassthroughFunction(const InputStreamShardSet& inputs,
-                                      OutputStreamShardSet* outputs) {
+absl::Status PassthroughFunction(const InputStreamShardSet& inputs,
+                                 OutputStreamShardSet* outputs) {
   for (int i = 0; i < inputs.NumEntries(); ++i) {
     if (!inputs.Index(i).Value().IsEmpty()) {
       outputs->Index(i).AddPacket(inputs.Index(i).Value());
     }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 // Tests demonstrating an FlowLimiterCalculator operating in a cyclic graph.
@@ -111,8 +111,8 @@ class FlowLimiterCalculatorSemaphoreTest : public testing::Test {
                            {"callback_1", Adopt(new auto(semaphore_1_func))},
                        }));
 
-    allow_poller_.reset(new OutputStreamPoller(
-        graph_.AddOutputStreamPoller("allow").ValueOrDie()));
+    allow_poller_.reset(
+        new OutputStreamPoller(graph_.AddOutputStreamPoller("allow").value()));
   }
 
   // Adds a packet to a graph input stream.
@@ -203,22 +203,22 @@ TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) {
 // A calculator that sleeps during Process.
 class SleepCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag("PACKET").SetAny();
     cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
     cc->InputSidePackets().Tag("SLEEP_TIME").Set<int64>();
     cc->InputSidePackets().Tag("WARMUP_TIME").Set<int64>();
     cc->InputSidePackets().Tag("CLOCK").Set<mediapipe::Clock*>();
     cc->SetTimestampOffset(0);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     clock_ = cc->InputSidePackets().Tag("CLOCK").Get<mediapipe::Clock*>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     ++packet_count;
     absl::Duration sleep_time = absl::Microseconds(
         packet_count == 1
@@ -226,7 +226,7 @@ class SleepCalculator : public CalculatorBase {
             : cc->InputSidePackets().Tag("SLEEP_TIME").Get<int64>());
     clock_->Sleep(sleep_time);
     cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:
@@ -239,15 +239,15 @@ REGISTER_CALCULATOR(SleepCalculator);
 // Drops the 3rd packet, and optionally the corresponding timestamp bound.
 class DropCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag("PACKET").SetAny();
     cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
     cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Set<bool>();
     cc->SetProcessTimestampBounds(true);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     if (!cc->Inputs().Tag("PACKET").Value().IsEmpty()) {
       ++packet_count;
     }
@@ -259,7 +259,7 @@ class DropCalculator : public CalculatorBase {
       cc->Outputs().Tag("PACKET").SetNextTimestampBound(
           cc->InputTimestamp().NextAllowedInStream());
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:
@@ -365,11 +365,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) {
   MP_ASSERT_OK(graph_.Initialize(graph_config));
   MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) {
     out_1_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) {
     allow_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   simulation_clock_->ThreadStart();
   MP_ASSERT_OK(graph_.StartRun(side_packets));
@@ -437,11 +437,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedLost) {
   MP_ASSERT_OK(graph_.Initialize(graph_config));
   MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) {
     out_1_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) {
     allow_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   simulation_clock_->ThreadStart();
   MP_ASSERT_OK(graph_.StartRun(side_packets));
@@ -501,11 +501,11 @@ TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) {
   MP_ASSERT_OK(graph_.Initialize(graph_config));
   MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) {
     out_1_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) {
     allow_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   simulation_clock_->ThreadStart();
   MP_ASSERT_OK(graph_.StartRun(side_packets));
@@ -596,16 +596,16 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
   MP_ASSERT_OK(graph_.Initialize(graph_config));
   MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) {
     out_1_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   std::vector<Packet> out_2_packets;
   MP_EXPECT_OK(graph_.ObserveOutputStream("in_2_sampled", [&](Packet p) {
     out_2_packets.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) {
     allow_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   simulation_clock_->ThreadStart();
   MP_ASSERT_OK(graph_.StartRun(side_packets));
@@ -705,16 +705,16 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
   MP_ASSERT_OK(graph_.Initialize(graph_config));
   MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) {
     out_1_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   std::vector<Packet> out_2_packets;
   MP_EXPECT_OK(graph_.ObserveOutputStream("in_2_sampled", [&](Packet p) {
     out_2_packets.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) {
     allow_packets_.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }));
   simulation_clock_->ThreadStart();
   MP_ASSERT_OK(graph_.StartRun(side_packets));

mediapipe/calculators/core/gate_calculator.cc

@@ -82,8 +82,7 @@ class GateCalculator : public CalculatorBase {
  public:
   GateCalculator() {}
 
-  static mediapipe::Status CheckAndInitAllowDisallowInputs(
-      CalculatorContract* cc) {
+  static absl::Status CheckAndInitAllowDisallowInputs(CalculatorContract* cc) {
     bool input_via_side_packet = cc->InputSidePackets().HasTag("ALLOW") ||
                                  cc->InputSidePackets().HasTag("DISALLOW");
     bool input_via_stream =
@@ -110,10 +109,10 @@ class GateCalculator : public CalculatorBase {
         cc->Inputs().Tag("DISALLOW").Set<bool>();
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc));
 
     const int num_data_streams = cc->Inputs().NumEntries("");
@@ -130,10 +129,10 @@ class GateCalculator : public CalculatorBase {
       cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
     }
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     use_side_packet_for_allow_disallow_ = false;
     if (cc->InputSidePackets().HasTag("ALLOW")) {
       use_side_packet_for_allow_disallow_ = true;
@@ -153,10 +152,10 @@ class GateCalculator : public CalculatorBase {
     const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
     empty_packets_as_allow_ = options.empty_packets_as_allow();
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     bool allow = empty_packets_as_allow_;
     if (use_side_packet_for_allow_disallow_) {
       allow = allow_by_side_packet_decision_;
@@ -195,7 +194,7 @@ class GateCalculator : public CalculatorBase {
           cc->Outputs().Get("", i).Close();
         }
       }
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     }
 
     // Process data streams.
@@ -205,7 +204,7 @@ class GateCalculator : public CalculatorBase {
       }
     }
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/gate_calculator_test.cc

@@ -25,7 +25,7 @@ namespace {
 class GateCalculatorTest : public ::testing::Test {
  protected:
   // Helper to run a graph and return status.
-  static mediapipe::Status RunGraph(const std::string& proto) {
+  static absl::Status RunGraph(const std::string& proto) {
     auto runner = absl::make_unique<CalculatorRunner>(
         ParseTextProtoOrDie<CalculatorGraphConfig::Node>(proto));
     return runner->Run();

mediapipe/calculators/core/immediate_mux_calculator.cc

@@ -43,16 +43,16 @@ class ImmediateMuxCalculator : public CalculatorBase {
  public:
   // This calculator combines any set of input streams into a single
   // output stream.  All input stream types must match the output stream type.
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  static absl::Status GetContract(CalculatorContract* cc);
 
   // Passes any input packet to the output stream immediately, unless the
   // packet timestamp is lower than a previously passed packet.
-  mediapipe::Status Process(CalculatorContext* cc) override;
-  mediapipe::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
 };
 REGISTER_CALCULATOR(ImmediateMuxCalculator);
 
-mediapipe::Status ImmediateMuxCalculator::GetContract(CalculatorContract* cc) {
+absl::Status ImmediateMuxCalculator::GetContract(CalculatorContract* cc) {
   RET_CHECK(cc->Outputs().NumEntries() >= 1 && cc->Outputs().NumEntries() <= 2)
       << "This calculator produces only one or two output streams.";
   cc->Outputs().Index(0).SetAny();
@@ -62,15 +62,15 @@ mediapipe::Status ImmediateMuxCalculator::GetContract(CalculatorContract* cc) {
   for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
     cc->Inputs().Index(i).SetSameAs(&cc->Outputs().Index(0));
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status ImmediateMuxCalculator::Open(CalculatorContext* cc) {
+absl::Status ImmediateMuxCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
+absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
   // Pass along the first packet, unless it has been superseded.
   for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
     const Packet& packet = cc->Inputs().Index(i).Value();
@@ -88,7 +88,7 @@ mediapipe::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
       }
     }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/calculators/core/immediate_mux_calculator_test.cc

@@ -289,19 +289,19 @@ TEST_F(ImmediateMuxCalculatorTest, SimultaneousTimestamps) {
 }
 
 // A Calculator::Process callback function.
-typedef std::function<mediapipe::Status(const InputStreamShardSet&,
-                                        OutputStreamShardSet*)>
+typedef std::function<absl::Status(const InputStreamShardSet&,
+                                   OutputStreamShardSet*)>
     ProcessFunction;
 
 // A testing callback function that passes through all packets.
-mediapipe::Status PassThrough(const InputStreamShardSet& inputs,
-                              OutputStreamShardSet* outputs) {
+absl::Status PassThrough(const InputStreamShardSet& inputs,
+                         OutputStreamShardSet* outputs) {
   for (int i = 0; i < inputs.NumEntries(); ++i) {
     if (!inputs.Index(i).Value().IsEmpty()) {
       outputs->Index(i).AddPacket(inputs.Index(i).Value());
     }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 TEST_F(ImmediateMuxCalculatorTest, Demux) {
@@ -325,7 +325,7 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
   auto out_cb = [&](const Packet& p) {
     absl::MutexLock lock(&out_mutex);
     out_packets.push_back(p);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   };
   auto wait_for = [&](std::function<bool()> cond) {
     absl::MutexLock lock(&out_mutex);

mediapipe/calculators/core/make_pair_calculator.cc

@@ -15,10 +15,12 @@
 #include <utility>
 #include <vector>
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // Given two input streams (A, B), output a single stream containing a pair<A,
 // B>.
@@ -30,32 +32,27 @@ namespace mediapipe {
 //   input_stream: "packet_b"
 //   output_stream: "output_pair_a_b"
 // }
-class MakePairCalculator : public CalculatorBase {
+class MakePairCalculator : public Node {
  public:
-  MakePairCalculator() {}
-  ~MakePairCalculator() override {}
+  static constexpr Input<AnyType>::Multiple kIn{""};
+  // Note that currently api2::Packet is a different type from mediapipe::Packet
+  static constexpr Output<std::pair<mediapipe::Packet, mediapipe::Packet>>
+      kPair{""};
 
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Index(0).SetAny();
-    cc->Inputs().Index(1).SetAny();
-    cc->Outputs().Index(0).Set<std::pair<Packet, Packet>>();
-    return mediapipe::OkStatus();
+  MEDIAPIPE_NODE_CONTRACT(kIn, kPair);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_EQ(kIn(cc).Count(), 2);
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
-  }
-
-  mediapipe::Status Process(CalculatorContext* cc) override {
-    cc->Outputs().Index(0).Add(
-        new std::pair<Packet, Packet>(cc->Inputs().Index(0).Value(),
-                                      cc->Inputs().Index(1).Value()),
-        cc->InputTimestamp());
-    return mediapipe::OkStatus();
+  absl::Status Process(CalculatorContext* cc) override {
+    kPair(cc).Send({kIn(cc)[0].packet(), kIn(cc)[1].packet()});
+    return absl::OkStatus();
   }
 };
 
-REGISTER_CALCULATOR(MakePairCalculator);
+MEDIAPIPE_REGISTER_NODE(MakePairCalculator);
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/matrix_multiply_calculator.cc

@@ -13,11 +13,13 @@
 // limitations under the License.
 
 #include "Eigen/Core"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+namespace api2 {
 // Perform a (left) matrix multiply.  Meaning (output = A * input)
 // where A is the matrix which is provided as an input side packet.
 //
@@ -28,39 +30,22 @@ namespace mediapipe {
 //   output_stream: "multiplied_samples"
 //   input_side_packet: "multiplication_matrix"
 // }
-class MatrixMultiplyCalculator : public CalculatorBase {
+class MatrixMultiplyCalculator : public Node {
  public:
-  MatrixMultiplyCalculator() {}
-  ~MatrixMultiplyCalculator() override {}
+  static constexpr Input<Matrix> kIn{""};
+  static constexpr Output<Matrix> kOut{""};
+  static constexpr SideInput<Matrix> kSide{""};
 
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kSide);
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 };
-REGISTER_CALCULATOR(MatrixMultiplyCalculator);
+MEDIAPIPE_REGISTER_NODE(MatrixMultiplyCalculator);
 
-// static
-mediapipe::Status MatrixMultiplyCalculator::GetContract(
-    CalculatorContract* cc) {
-  cc->Inputs().Index(0).Set<Matrix>();
-  cc->Outputs().Index(0).Set<Matrix>();
-  cc->InputSidePackets().Index(0).Set<Matrix>();
-  return mediapipe::OkStatus();
-}
-
-mediapipe::Status MatrixMultiplyCalculator::Open(CalculatorContext* cc) {
-  // The output is at the same timestamp as the input.
-  cc->SetOffset(TimestampDiff(0));
-  return mediapipe::OkStatus();
-}
-
-mediapipe::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
-  Matrix* multiplied = new Matrix();
-  *multiplied = cc->InputSidePackets().Index(0).Get<Matrix>() *
-                cc->Inputs().Index(0).Get<Matrix>();
-  cc->Outputs().Index(0).Add(multiplied, cc->InputTimestamp());
-  return mediapipe::OkStatus();
+absl::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
+  kOut(cc).Send(*kSide(cc) * *kIn(cc));
+  return absl::OkStatus();
 }
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/matrix_subtract_calculator.cc

@@ -13,11 +13,13 @@
 // limitations under the License.
 
 #include "Eigen/Core"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // Subtract input matrix from the side input matrix and vice versa. The matrices
 // must have the same dimension.
@@ -41,83 +43,39 @@ namespace mediapipe {
 //   input_side_packet: "MINUEND:side_matrix"
 //   output_stream: "output_matrix"
 // }
-class MatrixSubtractCalculator : public CalculatorBase {
+class MatrixSubtractCalculator : public Node {
  public:
-  MatrixSubtractCalculator() {}
-  ~MatrixSubtractCalculator() override {}
+  static constexpr Input<Matrix>::SideFallback kMinuend{"MINUEND"};
+  static constexpr Input<Matrix>::SideFallback kSubtrahend{"SUBTRAHEND"};
+  static constexpr Output<Matrix> kOut{""};
 
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  MEDIAPIPE_NODE_CONTRACT(kMinuend, kSubtrahend, kOut);
+  static absl::Status UpdateContract(CalculatorContract* cc);
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  bool subtract_from_input_ = false;
+  absl::Status Process(CalculatorContext* cc) override;
 };
-REGISTER_CALCULATOR(MatrixSubtractCalculator);
+MEDIAPIPE_REGISTER_NODE(MatrixSubtractCalculator);
 
 // static
-mediapipe::Status MatrixSubtractCalculator::GetContract(
-    CalculatorContract* cc) {
-  if (cc->Inputs().NumEntries() != 1 ||
-      cc->InputSidePackets().NumEntries() != 1) {
-    return mediapipe::InvalidArgumentError(
-        "MatrixSubtractCalculator only accepts exactly one input stream and "
-        "one "
-        "input side packet");
-  }
-  if (cc->Inputs().HasTag("MINUEND") &&
-      cc->InputSidePackets().HasTag("SUBTRAHEND")) {
-    cc->Inputs().Tag("MINUEND").Set<Matrix>();
-    cc->InputSidePackets().Tag("SUBTRAHEND").Set<Matrix>();
-  } else if (cc->Inputs().HasTag("SUBTRAHEND") &&
-             cc->InputSidePackets().HasTag("MINUEND")) {
-    cc->Inputs().Tag("SUBTRAHEND").Set<Matrix>();
-    cc->InputSidePackets().Tag("MINUEND").Set<Matrix>();
-  } else {
-    return mediapipe::InvalidArgumentError(
-        "Must specify exactly one minuend and one subtrahend.");
-  }
-  cc->Outputs().Index(0).Set<Matrix>();
-  return mediapipe::OkStatus();
+absl::Status MatrixSubtractCalculator::UpdateContract(CalculatorContract* cc) {
+  // TODO: the next restriction could be relaxed.
+  RET_CHECK(kMinuend(cc).IsStream() ^ kSubtrahend(cc).IsStream())
+      << "MatrixSubtractCalculator only accepts exactly one input stream and "
+         "one input side packet";
+  return absl::OkStatus();
 }
 
-mediapipe::Status MatrixSubtractCalculator::Open(CalculatorContext* cc) {
-  // The output is at the same timestamp as the input.
-  cc->SetOffset(TimestampDiff(0));
-  if (cc->Inputs().HasTag("MINUEND")) {
-    subtract_from_input_ = true;
+absl::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
+  const Matrix& minuend = *kMinuend(cc);
+  const Matrix& subtrahend = *kSubtrahend(cc);
+  if (minuend.rows() != subtrahend.rows() ||
+      minuend.cols() != subtrahend.cols()) {
+    return absl::InvalidArgumentError(
+        "Minuend and subtrahend must have the same dimensions.");
   }
-  return mediapipe::OkStatus();
-}
-
-mediapipe::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
-  Matrix* subtracted = new Matrix();
-  if (subtract_from_input_) {
-    const Matrix& input_matrix = cc->Inputs().Tag("MINUEND").Get<Matrix>();
-    const Matrix& side_input_matrix =
-        cc->InputSidePackets().Tag("SUBTRAHEND").Get<Matrix>();
-    if (input_matrix.rows() != side_input_matrix.rows() ||
-        input_matrix.cols() != side_input_matrix.cols()) {
-      return mediapipe::InvalidArgumentError(
-          "Input matrix and the input side matrix must have the same "
-          "dimension.");
-    }
-    *subtracted = input_matrix - side_input_matrix;
-  } else {
-    const Matrix& input_matrix = cc->Inputs().Tag("SUBTRAHEND").Get<Matrix>();
-    const Matrix& side_input_matrix =
-        cc->InputSidePackets().Tag("MINUEND").Get<Matrix>();
-    if (input_matrix.rows() != side_input_matrix.rows() ||
-        input_matrix.cols() != side_input_matrix.cols()) {
-      return mediapipe::InvalidArgumentError(
-          "Input matrix and the input side matrix must have the same "
-          "dimension.");
-    }
-    *subtracted = side_input_matrix - input_matrix;
-  }
-  cc->Outputs().Index(0).Add(subtracted, cc->InputTimestamp());
-  return mediapipe::OkStatus();
+  kOut(cc).Send(minuend - subtrahend);
+  return absl::OkStatus();
 }
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -89,9 +89,8 @@ TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
       )");
   CalculatorRunner runner(node_config);
   auto status = runner.Run();
-  EXPECT_THAT(
-      status.message(),
-      testing::HasSubstr("specify exactly one minuend and one subtrahend."));
+  EXPECT_THAT(status.message(), testing::HasSubstr("must be connected"));
+  EXPECT_THAT(status.message(), testing::HasSubstr("not both"));
 }
 
 TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {

mediapipe/calculators/core/matrix_to_vector_calculator.cc

@@ -21,6 +21,7 @@
 
 #include "Eigen/Core"
 #include "absl/memory/memory.h"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -30,6 +31,7 @@
 #include "mediapipe/util/time_series_util.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // A calculator that converts a Matrix M to a vector containing all the
 // entries of M in column-major order.
@@ -40,33 +42,27 @@ namespace mediapipe {
 //   input_stream: "input_matrix"
 //   output_stream: "column_major_vector"
 // }
-class MatrixToVectorCalculator : public CalculatorBase {
+class MatrixToVectorCalculator : public Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Index(0).Set<Matrix>(
-        // Input Packet containing a Matrix.
-    );
-    cc->Outputs().Index(0).Set<std::vector<float>>(
-        // Output Packet containing a vector, one for each input Packet.
-    );
-    return mediapipe::OkStatus();
-  }
+  static constexpr Input<Matrix> kIn{""};
+  static constexpr Output<std::vector<float>> kOut{""};
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Open(CalculatorContext* cc) override;
 
   // Outputs a packet containing a vector for each input packet.
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 };
-REGISTER_CALCULATOR(MatrixToVectorCalculator);
+MEDIAPIPE_REGISTER_NODE(MatrixToVectorCalculator);
 
-mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
-  // Inform the framework that we don't alter timestamps.
-  cc->SetOffset(mediapipe::TimestampDiff(0));
+absl::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(0);
   return mediapipe::OkStatus();
 }
 
-mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
-  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+absl::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
+  const Matrix& input = *kIn(cc);
   auto output = absl::make_unique<std::vector<float>>();
 
   // The following lines work to convert the Matrix to a vector because Matrix
@@ -76,8 +72,9 @@ mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
       Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
   output_as_matrix = input;
 
-  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
-  return mediapipe::OkStatus();
+  kOut(cc).Send(std::move(output));
+  return absl::OkStatus();
 }
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/merge_calculator.cc

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // This calculator takes a set of input streams and combines them into a single
 // output stream. The packets from different streams do not need to contain the
@@ -41,51 +43,43 @@ namespace mediapipe {
 //   output_stream: "merged_shot_infos"
 // }
 //
-class MergeCalculator : public CalculatorBase {
+class MergeCalculator : public Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK_GT(cc->Inputs().NumEntries(), 0)
-        << "Needs at least one input stream";
-    RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
-    if (cc->Inputs().NumEntries() == 1) {
+  static constexpr Input<AnyType>::Multiple kIn{""};
+  static constexpr Output<AnyType> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
+    if (kIn(cc).Count() == 1) {
       LOG(WARNING)
           << "MergeCalculator expects multiple input streams to merge but is "
              "receiving only one. Make sure the calculator is configured "
              "correctly or consider removing this calculator to reduce "
              "unnecessary overhead.";
     }
-
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      cc->Inputs().Index(i).SetAny();
-    }
-    cc->Outputs().Index(0).SetAny();
-
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
-    cc->SetOffset(TimestampDiff(0));
-
-    return mediapipe::OkStatus();
-  }
-
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     // Output the packet from the first input stream with a packet ready at this
     // timestamp.
-    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
-      if (!cc->Inputs().Index(i).IsEmpty()) {
-        cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(i).Value());
-        return mediapipe::OkStatus();
+    for (const auto& input : kIn(cc)) {
+      if (!input.IsEmpty()) {
+        kOut(cc).Send(input.packet());
+        return absl::OkStatus();
       }
     }
 
     LOG(WARNING) << "Empty input packets at timestamp "
                  << cc->InputTimestamp().Value();
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 
-REGISTER_CALCULATOR(MergeCalculator);
+MEDIAPIPE_REGISTER_NODE(MergeCalculator);
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/mux_calculator.cc

@@ -12,97 +12,45 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 
 namespace mediapipe {
-
-namespace {
-constexpr char kSelectTag[] = "SELECT";
-constexpr char kInputTag[] = "INPUT";
-}  // namespace
+namespace api2 {
 
 // A Calculator that selects an input stream from "INPUT:0", "INPUT:1", ...,
-// using the integer value (0, 1, ...) in the packet on the kSelectTag input
+// using the integer value (0, 1, ...) in the packet on the "SELECT" input
 // stream, and passes the packet on the selected input stream to the "OUTPUT"
 // output stream.
-// The kSelectTag input can also be passed in as an input side packet, instead
-// of as an input stream. Either of input stream or input side packet must be
-// specified but not both.
 //
 // Note that this calculator defaults to use MuxInputStreamHandler, which is
 // required for this calculator. However, it can be overridden to work with
 // other InputStreamHandlers. Check out the unit tests on for an example usage
 // with DefaultInputStreamHandler.
-class MuxCalculator : public CalculatorBase {
+// TODO: why would you need to use DefaultISH? Perhaps b/167596925?
+class MuxCalculator : public Node {
  public:
-  static mediapipe::Status CheckAndInitAllowDisallowInputs(
-      CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag(kSelectTag) ^
-              cc->InputSidePackets().HasTag(kSelectTag));
-    if (cc->Inputs().HasTag(kSelectTag)) {
-      cc->Inputs().Tag(kSelectTag).Set<int>();
-    } else {
-      cc->InputSidePackets().Tag(kSelectTag).Set<int>();
+  static constexpr Input<int>::SideFallback kSelect{"SELECT"};
+  // TODO: this currently sets them all to Any independently, instead
+  // of the first being Any and the others being SameAs.
+  static constexpr Input<AnyType>::Multiple kIn{"INPUT"};
+  static constexpr Output<SameType<kIn>> kOut{"OUTPUT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kSelect, kIn, kOut,
+                          StreamHandler("MuxInputStreamHandler"));
+
+  absl::Status Process(CalculatorContext* cc) final {
+    int select = *kSelect(cc);
+    RET_CHECK(0 <= select && select < kIn(cc).Count());
+    if (!kIn(cc)[select].IsEmpty()) {
+      kOut(cc).Send(kIn(cc)[select].packet());
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
-
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc));
-    CollectionItemId data_input_id = cc->Inputs().BeginId(kInputTag);
-    PacketType* data_input0 = &cc->Inputs().Get(data_input_id);
-    data_input0->SetAny();
-    ++data_input_id;
-    for (; data_input_id < cc->Inputs().EndId(kInputTag); ++data_input_id) {
-      cc->Inputs().Get(data_input_id).SetSameAs(data_input0);
-    }
-    RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
-    cc->Outputs().Tag("OUTPUT").SetSameAs(data_input0);
-
-    cc->SetInputStreamHandler("MuxInputStreamHandler");
-    MediaPipeOptions options;
-    cc->SetInputStreamHandlerOptions(options);
-
-    return mediapipe::OkStatus();
-  }
-
-  mediapipe::Status Open(CalculatorContext* cc) final {
-    use_side_packet_select_ = false;
-    if (cc->InputSidePackets().HasTag(kSelectTag)) {
-      use_side_packet_select_ = true;
-      selected_index_ = cc->InputSidePackets().Tag(kSelectTag).Get<int>();
-    } else {
-      select_input_ = cc->Inputs().GetId(kSelectTag, 0);
-    }
-    data_input_base_ = cc->Inputs().GetId(kInputTag, 0);
-    num_data_inputs_ = cc->Inputs().NumEntries(kInputTag);
-    output_ = cc->Outputs().GetId("OUTPUT", 0);
-    cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
-  }
-
-  mediapipe::Status Process(CalculatorContext* cc) final {
-    int select = use_side_packet_select_
-                     ? selected_index_
-                     : cc->Inputs().Get(select_input_).Get<int>();
-    RET_CHECK(0 <= select && select < num_data_inputs_);
-    if (!cc->Inputs().Get(data_input_base_ + select).IsEmpty()) {
-      cc->Outputs().Get(output_).AddPacket(
-          cc->Inputs().Get(data_input_base_ + select).Value());
-    }
-    return mediapipe::OkStatus();
-  }
-
- private:
-  CollectionItemId select_input_;
-  CollectionItemId data_input_base_;
-  int num_data_inputs_ = 0;
-  CollectionItemId output_;
-  bool use_side_packet_select_;
-  int selected_index_;
 };
 
-REGISTER_CALCULATOR(MuxCalculator);
+MEDIAPIPE_REGISTER_NODE(MuxCalculator);
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/mux_calculator_test.cc

@@ -134,7 +134,7 @@ void RunGraph(const std::string& graph_config_proto,
               const std::string& input_stream_name, int num_input_packets,
               std::function<Packet(int)> input_fn,
               const std::string& output_stream_name,
-              std::function<mediapipe::Status(const Packet&)> output_fn) {
+              std::function<absl::Status(const Packet&)> output_fn) {
   CalculatorGraphConfig config =
       mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_config_proto);
   CalculatorGraph graph;
@@ -165,9 +165,9 @@ TEST(MuxCalculatorTest, InputStreamSelector_DefaultInputStreamHandler) {
   // Output and handling.
   std::vector<int> output;
   // This function collects the output from the packet.
-  auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
+  auto output_fn = [&output](const Packet& p) -> absl::Status {
     output.push_back(p.Get<int>());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   };
 
   RunGraph(kTestGraphConfig1, {}, kInputName, input_packets.size(), input_fn,
@@ -191,9 +191,9 @@ TEST(MuxCalculatorTest, InputSidePacketSelector_DefaultInputStreamHandler) {
   // Output and handling.
   std::vector<int> output;
   // This function collects the output from the packet.
-  auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
+  auto output_fn = [&output](const Packet& p) -> absl::Status {
     output.push_back(p.Get<int>());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   };
 
   RunGraph(kTestGraphConfig2, {{kInputSelector, MakePacket<int>(0)}},
@@ -225,9 +225,9 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
   // Output and handling.
   std::vector<int> output;
   // This function collects the output from the packet.
-  auto output_fn = [&output](const Packet& p) -> mediapipe::Status {
+  auto output_fn = [&output](const Packet& p) -> absl::Status {
     output.push_back(p.Get<int>());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   };
 
   RunGraph(kTestGraphConfig3, {}, kInputName, input_packets.size(), input_fn,
@@ -260,7 +260,7 @@ TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
   MP_ASSERT_OK(
       graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
         output = p.Get<std::shared_ptr<int>>();
-        return mediapipe::OkStatus();
+        return absl::OkStatus();
       }));
 
   MP_ASSERT_OK(graph.StartRun({}));

mediapipe/calculators/core/nonzero_calculator.cc

@@ -0,0 +1,42 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A Calculator that returns 0 if INPUT is 0, and 1 otherwise.
+class NonZeroCalculator : public Node {
+ public:
+  static constexpr Input<int>::SideFallback kIn{"INPUT"};
+  static constexpr Output<int> kOut{"OUTPUT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      auto output = std::make_unique<int>((*kIn(cc) != 0) ? 1 : 0);
+      kOut(cc).Send(std::move(output));
+    }
+    return absl::OkStatus();
+  }
+};
+
+MEDIAPIPE_REGISTER_NODE(NonZeroCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/packet_cloner_calculator.cc

@@ -45,17 +45,17 @@ namespace mediapipe {
 //   packet_inner_join_calculator.cc: Don't output unless all inputs are new.
 class PacketClonerCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     const int tick_signal_index = cc->Inputs().NumEntries() - 1;
     for (int i = 0; i < tick_signal_index; ++i) {
       cc->Inputs().Index(i).SetAny();
       cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
     }
     cc->Inputs().Index(tick_signal_index).SetAny();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     // Load options.
     const auto calculator_options =
         cc->Options<mediapipe::PacketClonerCalculatorOptions>();
@@ -71,10 +71,10 @@ class PacketClonerCalculator : public CalculatorBase {
         cc->Outputs().Index(i).SetHeader(cc->Inputs().Index(i).Header());
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     // Store input signals.
     for (int i = 0; i < tick_signal_index_; ++i) {
       if (!cc->Inputs().Index(i).Value().IsEmpty()) {
@@ -88,7 +88,7 @@ class PacketClonerCalculator : public CalculatorBase {
         // Return if one of the input is null.
         for (int i = 0; i < tick_signal_index_; ++i) {
           if (current_[i].IsEmpty()) {
-            return mediapipe::OkStatus();
+            return absl::OkStatus();
           }
         }
       }
@@ -103,7 +103,7 @@ class PacketClonerCalculator : public CalculatorBase {
         }
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/packet_inner_join_calculator.cc

@@ -34,10 +34,10 @@ namespace mediapipe {
 //   packet_cloner_calculator.cc: Repeats last-seen packets from empty inputs.
 class PacketInnerJoinCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  static absl::Status GetContract(CalculatorContract* cc);
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 
  private:
   int num_streams_;
@@ -45,8 +45,7 @@ class PacketInnerJoinCalculator : public CalculatorBase {
 
 REGISTER_CALCULATOR(PacketInnerJoinCalculator);
 
-mediapipe::Status PacketInnerJoinCalculator::GetContract(
-    CalculatorContract* cc) {
+absl::Status PacketInnerJoinCalculator::GetContract(CalculatorContract* cc) {
   RET_CHECK(cc->Inputs().NumEntries() == cc->Outputs().NumEntries())
       << "The number of input and output streams must match.";
   const int num_streams = cc->Inputs().NumEntries();
@@ -54,25 +53,25 @@ mediapipe::Status PacketInnerJoinCalculator::GetContract(
     cc->Inputs().Index(i).SetAny();
     cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
+absl::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
   num_streams_ = cc->Inputs().NumEntries();
   cc->SetOffset(TimestampDiff(0));
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
+absl::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
   for (int i = 0; i < num_streams_; ++i) {
     if (cc->Inputs().Index(i).Value().IsEmpty()) {
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     }
   }
   for (int i = 0; i < num_streams_; ++i) {
     cc->Outputs().Index(i).AddPacket(cc->Inputs().Index(i).Value());
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 }  // namespace mediapipe

mediapipe/calculators/core/packet_presence_calculator.cc

@@ -57,26 +57,26 @@ namespace mediapipe {
 // }
 class PacketPresenceCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag("PACKET").SetAny();
     cc->Outputs().Tag("PRESENCE").Set<bool>();
     // Process() function is invoked in response to input stream timestamp
     // bound updates.
     cc->SetProcessTimestampBounds(true);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override {
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     cc->Outputs()
         .Tag("PRESENCE")
         .AddPacket(MakePacket<bool>(!cc->Inputs().Tag("PACKET").IsEmpty())
                        .At(cc->InputTimestamp()));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 REGISTER_CALCULATOR(PacketPresenceCalculator);

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -47,8 +47,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 }
 }  // namespace
 
-mediapipe::Status PacketResamplerCalculator::GetContract(
-    CalculatorContract* cc) {
+absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) {
   const auto& resampler_options =
       cc->Options<PacketResamplerCalculatorOptions>();
   if (cc->InputSidePackets().HasTag("OPTIONS")) {
@@ -78,17 +77,16 @@ mediapipe::Status PacketResamplerCalculator::GetContract(
     RET_CHECK(cc->InputSidePackets().HasTag("SEED"));
     cc->InputSidePackets().Tag("SEED").Set<std::string>();
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
+absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
   const auto resampler_options =
       tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
                             cc->InputSidePackets(), "OPTIONS");
 
   flush_last_packet_ = resampler_options.flush_last_packet();
   jitter_ = resampler_options.jitter();
-  jitter_with_reflection_ = resampler_options.jitter_with_reflection();
 
   input_data_id_ = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id_.IsValid()) {
@@ -99,11 +97,7 @@ mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     output_data_id_ = cc->Outputs().GetId("", 0);
   }
 
-  period_count_ = 0;
   frame_rate_ = resampler_options.frame_rate();
-  base_timestamp_ = resampler_options.has_base_timestamp()
-                        ? Timestamp(resampler_options.base_timestamp())
-                        : Timestamp::Unset();
   start_time_ = resampler_options.has_start_time()
                     ? Timestamp(resampler_options.start_time())
                     : Timestamp::Min();
@@ -142,215 +136,63 @@ mediapipe::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
     }
   }
 
-  if (jitter_ != 0.0) {
-    if (resampler_options.output_header() !=
-        PacketResamplerCalculatorOptions::NONE) {
-      LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
-                      "the actual value.";
-    }
-    if (flush_last_packet_) {
-      flush_last_packet_ = false;
-      LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
-                      "ignored, because we are adding jitter.";
-    }
-    const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
-    random_ = CreateSecureRandom(seed);
-    if (random_ == nullptr) {
-      return mediapipe::Status(
-          mediapipe::StatusCode::kInvalidArgument,
-          "SecureRandom is not available.  With \"jitter\" specified, "
-          "PacketResamplerCalculator processing cannot proceed.");
-    }
-    packet_reservoir_random_ = CreateSecureRandom(seed);
-  }
-  packet_reservoir_ =
-      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
-  return mediapipe::OkStatus();
+  strategy_ = GetSamplingStrategy(resampler_options);
+
+  return strategy_->Open(cc);
 }
 
-mediapipe::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
+absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
   if (cc->InputTimestamp() == Timestamp::PreStream() &&
       cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") &&
       !cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) {
     video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get<VideoHeader>();
     video_header_.frame_rate = frame_rate_;
     if (cc->Inputs().Get(input_data_id_).IsEmpty()) {
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     }
   }
-  if (jitter_ != 0.0 && random_ != nullptr) {
-    // Packet reservior is used to make sure there's an output for every period,
-    // e.g. partial period at the end of the stream.
-    if (packet_reservoir_->IsEnabled() &&
-        (first_timestamp_ == Timestamp::Unset() ||
-         (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
-      auto curr_packet = cc->Inputs().Get(input_data_id_).Value();
-      packet_reservoir_->AddSample(curr_packet);
-    }
-    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
-  } else {
-    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
+
+  if (absl::Status status = strategy_->Process(cc); !status.ok()) {
+    return status;  // Avoid MP_RETURN_IF_ERROR macro for external release.
   }
+
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
-  return mediapipe::OkStatus();
+
+  return absl::OkStatus();
 }
 
-void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
-  next_output_timestamp_min_ = first_timestamp_;
-  if (jitter_with_reflection_) {
-    next_output_timestamp_ =
-        first_timestamp_ + random_->UnbiasedUniform64(frame_time_usec_);
-    return;
-  }
-  next_output_timestamp_ =
-      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
-}
-
-void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
-  packet_reservoir_->Clear();
-  if (jitter_with_reflection_) {
-    next_output_timestamp_min_ += frame_time_usec_;
-    Timestamp next_output_timestamp_max_ =
-        next_output_timestamp_min_ + frame_time_usec_;
-
-    next_output_timestamp_ += frame_time_usec_ +
-                              random_->UnbiasedUniform64(2 * jitter_usec_ + 1) -
-                              jitter_usec_;
-    next_output_timestamp_ = Timestamp(ReflectBetween(
-        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
-        next_output_timestamp_max_.Value()));
-    CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
-    CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
-    return;
-  }
-  packet_reservoir_->Disable();
-  next_output_timestamp_ +=
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
-}
-
-mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_NE(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    first_timestamp_ = cc->InputTimestamp();
-    InitializeNextOutputTimestampWithJitter();
-    if (first_timestamp_ == next_output_timestamp_) {
-      OutputWithinLimits(
-          cc,
-          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
-      UpdateNextOutputTimestampWithJitter();
-    }
-    return mediapipe::OkStatus();
-  }
-
-  if (frame_time_usec_ <
-      (cc->InputTimestamp() - last_packet_.Timestamp()).Value()) {
-    LOG_FIRST_N(WARNING, 2)
-        << "Adding jitter is not very useful when upsampling.";
-  }
-
-  while (true) {
-    const int64 last_diff =
-        (next_output_timestamp_ - last_packet_.Timestamp()).Value();
-    RET_CHECK_GT(last_diff, 0);
-    const int64 curr_diff =
-        (next_output_timestamp_ - cc->InputTimestamp()).Value();
-    if (curr_diff > 0) {
-      break;
-    }
-    OutputWithinLimits(cc, (std::abs(curr_diff) > last_diff
-                                ? last_packet_
-                                : cc->Inputs().Get(input_data_id_).Value())
-                               .At(next_output_timestamp_));
-    UpdateNextOutputTimestampWithJitter();
-  }
-  return mediapipe::OkStatus();
-}
-
-mediapipe::Status PacketResamplerCalculator::ProcessWithoutJitter(
-    CalculatorContext* cc) {
-  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
-  RET_CHECK_EQ(jitter_, 0.0);
-
-  if (first_timestamp_ == Timestamp::Unset()) {
-    // This is the first packet, initialize the first_timestamp_.
-    if (base_timestamp_ == Timestamp::Unset()) {
-      // Initialize first_timestamp_ with exactly the first packet timestamp.
-      first_timestamp_ = cc->InputTimestamp();
-    } else {
-      // Initialize first_timestamp_ with the first packet timestamp
-      // aligned to the base_timestamp_.
-      int64 first_index = MathUtil::SafeRound<int64, double>(
-          (cc->InputTimestamp() - base_timestamp_).Seconds() * frame_rate_);
-      first_timestamp_ =
-          base_timestamp_ + TimestampDiffFromSeconds(first_index / frame_rate_);
-    }
-    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
-      cc->Outputs()
-          .Tag("VIDEO_HEADER")
-          .Add(new VideoHeader(video_header_), Timestamp::PreStream());
-    }
-  }
-  const Timestamp received_timestamp = cc->InputTimestamp();
-  const int64 received_timestamp_idx =
-      TimestampToPeriodIndex(received_timestamp);
-  // Only consider the received packet if it belongs to the current period
-  // (== period_count_) or to a newer one (> period_count_).
-  if (received_timestamp_idx >= period_count_) {
-    // Fill the empty periods until we are in the same index as the received
-    // packet.
-    while (received_timestamp_idx > period_count_) {
-      OutputWithinLimits(
-          cc, last_packet_.At(PeriodIndexToTimestamp(period_count_)));
-      ++period_count_;
-    }
-    // Now, if the received packet has a timestamp larger than the middle of
-    // the current period, we can send a packet without waiting. We send the
-    // one closer to the middle.
-    Timestamp target_timestamp = PeriodIndexToTimestamp(period_count_);
-    if (received_timestamp >= target_timestamp) {
-      bool have_last_packet = (last_packet_.Timestamp() != Timestamp::Unset());
-      bool send_current =
-          !have_last_packet || (received_timestamp - target_timestamp <=
-                                target_timestamp - last_packet_.Timestamp());
-      if (send_current) {
-        OutputWithinLimits(
-            cc, cc->Inputs().Get(input_data_id_).Value().At(target_timestamp));
-      } else {
-        OutputWithinLimits(cc, last_packet_.At(target_timestamp));
-      }
-      ++period_count_;
-    }
-    // TODO: Add a mechanism to the framework to allow these packets
-    // to be output earlier (without waiting for a much later packet to
-    // arrive)
-
-    // Update the bound for the next packet.
-    cc->Outputs()
-        .Get(output_data_id_)
-        .SetNextTimestampBound(PeriodIndexToTimestamp(period_count_));
-  }
-  return mediapipe::OkStatus();
-}
-
-mediapipe::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
+absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) {
   if (!cc->GraphStatus().ok()) {
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
-  // Emit the last packet received if we have at least one packet, but
-  // haven't sent anything for its period.
-  if (first_timestamp_ != Timestamp::Unset() && flush_last_packet_ &&
-      TimestampToPeriodIndex(last_packet_.Timestamp()) == period_count_) {
-    OutputWithinLimits(cc,
-                       last_packet_.At(PeriodIndexToTimestamp(period_count_)));
+
+  return strategy_->Close(cc);
+}
+
+std::unique_ptr<PacketResamplerStrategy>
+PacketResamplerCalculator::GetSamplingStrategy(
+    const PacketResamplerCalculatorOptions& options) {
+  if (options.reproducible_sampling()) {
+    if (!options.jitter_with_reflection()) {
+      LOG(WARNING)
+          << "reproducible_sampling enabled w/ jitter_with_reflection "
+             "disabled. "
+          << "reproducible_sampling always uses jitter with reflection, "
+          << "Ignoring jitter_with_reflection setting.";
+    }
+    return absl::make_unique<ReproducibleJitterWithReflectionStrategy>(this);
   }
-  if (!packet_reservoir_->IsEmpty()) {
-    OutputWithinLimits(cc, packet_reservoir_->GetSample());
+
+  if (options.jitter() == 0) {
+    return absl::make_unique<NoJitterStrategy>(this);
   }
-  return mediapipe::OkStatus();
+
+  if (options.jitter_with_reflection()) {
+    return absl::make_unique<LegacyJitterWithReflectionStrategy>(this);
+  }
+
+  // With jitter and no reflection.
+  return absl::make_unique<JitterWithoutReflectionStrategy>(this);
 }
 
 Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const {
@@ -376,4 +218,479 @@ void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc,
   }
 }
 
+absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      std::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    LOG(INFO) << "Emitting pack from reservoir.";
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status LegacyJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (packet_reservoir_->IsEnabled() &&
+      (first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (first_timestamp_ == Timestamp::Unset()) {
+    first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestampWithJitter();
+    if (first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestampWithJitter();
+    // From now on every time a packet is emitted the timestamp of the next
+    // packet becomes known; that timestamp is stored in next_output_timestamp_.
+    // The only exception to this rule is the packet emitted from Close() which
+    // can only happen when jitter_with_reflection is enabled but in this case
+    // next_output_timestamp_min_ is a non-decreasing lower bound of any
+    // subsequent packet.
+    const Timestamp timestamp_bound = next_output_timestamp_min_;
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(timestamp_bound);
+  }
+  return absl::OkStatus();
+}
+
+void LegacyJitterWithReflectionStrategy::
+    InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_min_ = first_timestamp_;
+  next_output_timestamp_ =
+      first_timestamp_ +
+      random_->UnbiasedUniform64(calculator_->frame_time_usec_);
+}
+
+void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
+  packet_reservoir_->Clear();
+  next_output_timestamp_min_ += calculator_->frame_time_usec_;
+  Timestamp next_output_timestamp_max_ =
+      next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+  next_output_timestamp_ +=
+      calculator_->frame_time_usec_ +
+      random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) -
+      calculator_->jitter_usec_;
+  next_output_timestamp_ = Timestamp(ReflectBetween(
+      next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+      next_output_timestamp_max_.Value()));
+  CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+  CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+}
+
+absl::Status ReproducibleJitterWithReflectionStrategy::Open(
+    CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Close(
+    CalculatorContext* cc) {
+  // If last packet is non-empty and a packet hasn't been emitted for this
+  // period, emit the last packet.
+  if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(next_output_timestamp_));
+  }
+  return absl::OkStatus();
+}
+absl::Status ReproducibleJitterWithReflectionStrategy::Process(
+    CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+
+  if (calculator_->last_packet_.IsEmpty()) {
+    // last_packet is empty, this is the first packet of the stream.
+
+    InitializeNextOutputTimestamp(current_packet.Timestamp());
+
+    // If next_output_timestamp_ happens to fall before current_packet, emit
+    // current packet.  Only a single packet can be emitted at the beginning
+    // of the stream.
+    if (next_output_timestamp_ < current_packet.Timestamp()) {
+      calculator_->OutputWithinLimits(
+          cc, current_packet.At(next_output_timestamp_));
+      packet_emitted_this_period_ = true;
+    }
+
+    return absl::OkStatus();
+  }
+
+  // Last packet is set, so we are mid-stream.
+  if (calculator_->frame_time_usec_ <
+      (current_packet.Timestamp() - calculator_->last_packet_.Timestamp())
+          .Value()) {
+    // Note, if the stream is upsampling, this could lead to the same packet
+    // being emitted twice.  Upsampling and jitter doesn't make much sense
+    // but does technically work.
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  // Since we may be upsampling, we need to iteratively advance the
+  // next_output_timestamp_ one period at a time until it reaches the period
+  // current_packet is in.  During this process, last_packet and/or
+  // current_packet may be repeatly emitted.
+
+  UpdateNextOutputTimestamp(current_packet.Timestamp());
+
+  while (!packet_emitted_this_period_ &&
+         next_output_timestamp_ <= current_packet.Timestamp()) {
+    // last_packet < next_output_timestamp_ <= current_packet,
+    // so emit the closest packet.
+    Packet packet_to_emit =
+        current_packet.Timestamp() - next_output_timestamp_ <
+                next_output_timestamp_ - calculator_->last_packet_.Timestamp()
+            ? current_packet
+            : calculator_->last_packet_;
+    calculator_->OutputWithinLimits(cc,
+                                    packet_to_emit.At(next_output_timestamp_));
+
+    packet_emitted_this_period_ = true;
+
+    // If we are upsampling, packet_emitted_this_period_ can be reset by
+    // the following UpdateNext and the loop will iterate.
+    UpdateNextOutputTimestamp(current_packet.Timestamp());
+  }
+
+  // Set the bounds on the output stream.  Note, if we emitted a packet
+  // above, it will already be set at next_output_timestamp_ + 1, in which
+  // case we have to skip setting it.
+  if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() <
+      next_output_timestamp_) {
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (next_output_timestamp_min_ != Timestamp::Unset()) {
+    return;
+  }
+
+  next_output_timestamp_min_ = Timestamp(0);
+  next_output_timestamp_ =
+      Timestamp(GetNextRandom(calculator_->frame_time_usec_));
+
+  // While the current timestamp is ahead of the max (i.e. min + frame_time),
+  // fast-forward.
+  while (current_timestamp >=
+         next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    packet_emitted_this_period_ = true;  // Force update...
+    UpdateNextOutputTimestamp(current_timestamp);
+  }
+}
+
+void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp(
+    Timestamp current_timestamp) {
+  if (packet_emitted_this_period_ &&
+      current_timestamp >=
+          next_output_timestamp_min_ + calculator_->frame_time_usec_) {
+    next_output_timestamp_min_ += calculator_->frame_time_usec_;
+    Timestamp next_output_timestamp_max_ =
+        next_output_timestamp_min_ + calculator_->frame_time_usec_;
+
+    next_output_timestamp_ += calculator_->frame_time_usec_ +
+                              GetNextRandom(2 * calculator_->jitter_usec_ + 1) -
+                              calculator_->jitter_usec_;
+    next_output_timestamp_ = Timestamp(ReflectBetween(
+        next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
+        next_output_timestamp_max_.Value()));
+
+    packet_emitted_this_period_ = false;
+  }
+}
+
+absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+
+  if (resampler_options.output_header() !=
+      PacketResamplerCalculatorOptions::NONE) {
+    LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
+                    "the actual value.";
+  }
+
+  if (calculator_->flush_last_packet_) {
+    LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
+                    "ignored, because we are adding jitter.";
+  }
+
+  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  random_ = CreateSecureRandom(seed);
+  if (random_ == nullptr) {
+    return absl::InvalidArgumentError(
+        "SecureRandom is not available.  With \"jitter\" specified, "
+        "PacketResamplerCalculator processing cannot proceed.");
+  }
+
+  packet_reservoir_random_ = CreateSecureRandom(seed);
+  packet_reservoir_ =
+      absl::make_unique<PacketReservoir>(packet_reservoir_random_.get());
+
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) {
+  if (!packet_reservoir_->IsEmpty()) {
+    calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
+  }
+  return absl::OkStatus();
+}
+absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  // Packet reservior is used to make sure there's an output for every period,
+  // e.g. partial period at the end of the stream.
+  if (packet_reservoir_->IsEnabled() &&
+      (calculator_->first_timestamp_ == Timestamp::Unset() ||
+       (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) {
+    auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value();
+    packet_reservoir_->AddSample(curr_packet);
+  }
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    calculator_->first_timestamp_ = cc->InputTimestamp();
+    InitializeNextOutputTimestamp();
+    if (calculator_->first_timestamp_ == next_output_timestamp_) {
+      calculator_->OutputWithinLimits(cc, cc->Inputs()
+                                              .Get(calculator_->input_data_id_)
+                                              .Value()
+                                              .At(next_output_timestamp_));
+      UpdateNextOutputTimestamp();
+    }
+    return absl::OkStatus();
+  }
+
+  if (calculator_->frame_time_usec_ <
+      (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
+    LOG_FIRST_N(WARNING, 2)
+        << "Adding jitter is not very useful when upsampling.";
+  }
+
+  while (true) {
+    const int64 last_diff =
+        (next_output_timestamp_ - calculator_->last_packet_.Timestamp())
+            .Value();
+    RET_CHECK_GT(last_diff, 0);
+    const int64 curr_diff =
+        (next_output_timestamp_ - cc->InputTimestamp()).Value();
+    if (curr_diff > 0) {
+      break;
+    }
+    calculator_->OutputWithinLimits(
+        cc, (std::abs(curr_diff) > last_diff
+                 ? calculator_->last_packet_
+                 : cc->Inputs().Get(calculator_->input_data_id_).Value())
+                .At(next_output_timestamp_));
+    UpdateNextOutputTimestamp();
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(next_output_timestamp_);
+  }
+  return absl::OkStatus();
+}
+
+void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() {
+  next_output_timestamp_min_ = calculator_->first_timestamp_;
+  next_output_timestamp_ = calculator_->first_timestamp_ +
+                           calculator_->frame_time_usec_ * random_->RandFloat();
+}
+
+void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() {
+  packet_reservoir_->Clear();
+  packet_reservoir_->Disable();
+  next_output_timestamp_ += calculator_->frame_time_usec_ *
+                            ((1.0 - calculator_->jitter_) +
+                             2.0 * calculator_->jitter_ * random_->RandFloat());
+}
+
+absl::Status NoJitterStrategy::Open(CalculatorContext* cc) {
+  const auto resampler_options =
+      tool::RetrieveOptions(cc->Options<PacketResamplerCalculatorOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+  base_timestamp_ = resampler_options.has_base_timestamp()
+                        ? Timestamp(resampler_options.base_timestamp())
+                        : Timestamp::Unset();
+
+  period_count_ = 0;
+
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Close(CalculatorContext* cc) {
+  // Emit the last packet received if we have at least one packet, but
+  // haven't sent anything for its period.
+  if (calculator_->first_timestamp_ != Timestamp::Unset() &&
+      calculator_->flush_last_packet_ &&
+      calculator_->TimestampToPeriodIndex(
+          calculator_->last_packet_.Timestamp()) == period_count_) {
+    calculator_->OutputWithinLimits(
+        cc, calculator_->last_packet_.At(
+                calculator_->PeriodIndexToTimestamp(period_count_)));
+  }
+  return absl::OkStatus();
+}
+absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
+  RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
+
+  if (calculator_->first_timestamp_ == Timestamp::Unset()) {
+    // This is the first packet, initialize the first_timestamp_.
+    if (base_timestamp_ == Timestamp::Unset()) {
+      // Initialize first_timestamp_ with exactly the first packet timestamp.
+      calculator_->first_timestamp_ = cc->InputTimestamp();
+    } else {
+      // Initialize first_timestamp_ with the first packet timestamp
+      // aligned to the base_timestamp_.
+      int64 first_index = MathUtil::SafeRound<int64, double>(
+          (cc->InputTimestamp() - base_timestamp_).Seconds() *
+          calculator_->frame_rate_);
+      calculator_->first_timestamp_ =
+          base_timestamp_ +
+          TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
+    }
+    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
+      cc->Outputs()
+          .Tag("VIDEO_HEADER")
+          .Add(new VideoHeader(calculator_->video_header_),
+               Timestamp::PreStream());
+    }
+  }
+  const Timestamp received_timestamp = cc->InputTimestamp();
+  const int64 received_timestamp_idx =
+      calculator_->TimestampToPeriodIndex(received_timestamp);
+  // Only consider the received packet if it belongs to the current period
+  // (== period_count_) or to a newer one (> period_count_).
+  if (received_timestamp_idx >= period_count_) {
+    // Fill the empty periods until we are in the same index as the received
+    // packet.
+    while (received_timestamp_idx > period_count_) {
+      calculator_->OutputWithinLimits(
+          cc, calculator_->last_packet_.At(
+                  calculator_->PeriodIndexToTimestamp(period_count_)));
+      ++period_count_;
+    }
+    // Now, if the received packet has a timestamp larger than the middle of
+    // the current period, we can send a packet without waiting. We send the
+    // one closer to the middle.
+    Timestamp target_timestamp =
+        calculator_->PeriodIndexToTimestamp(period_count_);
+    if (received_timestamp >= target_timestamp) {
+      bool have_last_packet =
+          (calculator_->last_packet_.Timestamp() != Timestamp::Unset());
+      bool send_current =
+          !have_last_packet ||
+          (received_timestamp - target_timestamp <=
+           target_timestamp - calculator_->last_packet_.Timestamp());
+      if (send_current) {
+        calculator_->OutputWithinLimits(cc,
+                                        cc->Inputs()
+                                            .Get(calculator_->input_data_id_)
+                                            .Value()
+                                            .At(target_timestamp));
+      } else {
+        calculator_->OutputWithinLimits(
+            cc, calculator_->last_packet_.At(target_timestamp));
+      }
+      ++period_count_;
+    }
+    // TODO: Add a mechanism to the framework to allow these packets
+    // to be output earlier (without waiting for a much later packet to
+    // arrive)
+
+    // Update the bound for the next packet.
+    cc->Outputs()
+        .Get(calculator_->output_data_id_)
+        .SetNextTimestampBound(
+            calculator_->PeriodIndexToTimestamp(period_count_));
+  }
+  return absl::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.h

@@ -55,7 +55,7 @@ class PacketReservoir {
 //     correspond to timestamp t.
 //   - The next packet is chosen randomly (uniform distribution) among frames
 //     that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate].
-//     - if jitter_with_reflection_ is true, the timestamp will be reflected
+//     - if jitter_with_reflection is true, the timestamp will be reflected
 //       against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate)
 //       so that its marginal distribution is uniform within this interval.
 //       In the formula, t_0 is the timestamp of the first sampled
@@ -66,6 +66,17 @@ class PacketReservoir {
 //     the resampling.  For Cloud ML Video Intelligence API, the hash of the
 //     input video should serve this purpose.  For YouTube, either video ID or
 //     content hex ID of the input video should do.
+//   - If reproducible_samping is true, care is taken to allow reproducible
+//     "mid-stream" sampling.  The calculator can be executed on a stream that
+//     doesn't start at the first period.  For instance, if the calculator
+//     is run on a 10 second stream it will produce the same set of samples
+//     as two runs of the calculator, the first with 3 seconds of input starting
+//     at time 0 and the second with 7 seconds of input starting at time +3s.
+//     - In order to guarantee the exact same samples, 1) the inputs must be
+//       aligned with the sampling period.  For instance, if the sampling rate
+//       is 2 frames per second, streams should be aligned on 0.5 second
+//       boundaries, and 2) the stream must include at least one extra packet
+//       before and after the second aligned sampling period.
 //
 // If jitter_ is not specified:
 //   - The first packet defines the first_timestamp of the output stream,
@@ -99,24 +110,11 @@ class PacketReservoir {
 //   packet_downsampler_calculator.cc: skips packets regardless of timestamps.
 class PacketResamplerCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc);
+  static absl::Status GetContract(CalculatorContract* cc);
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Close(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override;
-
- private:
-  // Calculates the first sampled timestamp that incorporates a jittering
-  // offset.
-  void InitializeNextOutputTimestampWithJitter();
-  // Calculates the next sampled timestamp that incorporates a jittering offset.
-  void UpdateNextOutputTimestampWithJitter();
-
-  // Logic for Process() when jitter_ != 0.0.
-  mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
-
-  // Logic for Process() when jitter_ == 0.0.
-  mediapipe::Status ProcessWithoutJitter(CalculatorContext* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
 
   // Given the current count of periods that have passed, this returns
   // the next valid timestamp of the middle point of the next period:
@@ -141,6 +139,16 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Outputs a packet if it is in range (start_time_, end_time_).
   void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const;
 
+ protected:
+  // Returns Sampling Strategy to use.
+  //
+  // Virtual to allow injection of testing strategies.
+  virtual std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& options);
+
+ private:
+  std::unique_ptr<class PacketResamplerStrategy> strategy_;
+
   // The timestamp of the first packet received.
   Timestamp first_timestamp_;
 
@@ -150,14 +158,6 @@ class PacketResamplerCalculator : public CalculatorBase {
   // Inverse of frame_rate_.
   int64 frame_time_usec_;
 
-  // Number of periods that have passed (= #packets sent to the output).
-  //
-  // Can only be used if jitter_ equals zero.
-  int64 period_count_;
-
-  // The last packet that was received.
-  Packet last_packet_;
-
   VideoHeader video_header_;
   // The "DATA" input stream.
   CollectionItemId input_data_id_;
@@ -165,23 +165,15 @@ class PacketResamplerCalculator : public CalculatorBase {
   CollectionItemId output_data_id_;
 
   // Indicator whether to flush last packet even if its timestamp is greater
-  // than the final stream timestamp.  Set to false when jitter_ is non-zero.
+  // than the final stream timestamp.
   bool flush_last_packet_;
 
-  // Jitter-related variables.
-  std::unique_ptr<RandomBase> random_;
   double jitter_ = 0.0;
-  bool jitter_with_reflection_;
-  int64 jitter_usec_;
-  Timestamp next_output_timestamp_;
-  // If jittering_with_reflection_ is true, next_output_timestamp_ will be
-  // kept within the interval
-  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
-  Timestamp next_output_timestamp_min_;
 
-  // If specified, output timestamps are aligned with base_timestamp.
-  // Otherwise, they are aligned with the first input timestamp.
-  Timestamp base_timestamp_;
+  int64 jitter_usec_;
+
+  // The last packet that was received.
+  Packet last_packet_;
 
   // If specified, only outputs at/after start_time are included.
   Timestamp start_time_;
@@ -191,15 +183,210 @@ class PacketResamplerCalculator : public CalculatorBase {
 
   // If set, the output timestamps nearest to start_time and end_time
   // are included in the output, even if the nearest timestamp is not
-  // between start_time and end_time.W
+  // between start_time and end_time.
   bool round_limits_;
 
+  // Allow strategies access to all internal calculator state.
+  //
+  // The calculator and strategies are intimiately tied together so this should
+  // not break encapsulation.
+  friend class LegacyJitterWithReflectionStrategy;
+  friend class ReproducibleJitterWithReflectionStrategy;
+  friend class JitterWithoutReflectionStrategy;
+  friend class NoJitterStrategy;
+};
+
+// Abstract class encapsulating sampling stategy.
+//
+// These are used solely by PacketResamplerCalculator, but are exposed here
+// to facilitate tests.
+class PacketResamplerStrategy {
+ public:
+  PacketResamplerStrategy(PacketResamplerCalculator* calculator)
+      : calculator_(calculator) {}
+  virtual ~PacketResamplerStrategy() = default;
+
+  // Delegate for CalculatorBase::Open.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Open(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Close.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Close(CalculatorContext* cc) = 0;
+  // Delegate for CalculatorBase::Process.  See CalculatorBase for relevant
+  // implementation considerations.
+  virtual absl::Status Process(CalculatorContext* cc) = 0;
+
+ protected:
+  // Calculator running strategy.
+  PacketResamplerCalculator* calculator_;
+};
+
+// Strategy that applies Jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+//
+// This applies the legacy jitter with reflection which doesn't allow
+// for reproducibility of sampling when starting mid-stream.  This is maintained
+// for backward compatibility.
+class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void InitializeNextOutputTimestampWithJitter();
+  void UpdateNextOutputTimestampWithJitter();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // The timestamp of the first packet received.
+  Timestamp first_timestamp_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
   // packet reservior used for sampling random packet out of partial
   // period when jitter is enabled
   std::unique_ptr<PacketReservoir> packet_reservoir_;
+
   // random number generator used in packet_reservior_.
   std::unique_ptr<RandomBase> packet_reservoir_random_;
 };
 
+// Strategy that applies reproducible jitter with reflection based sampling.
+//
+// Used by PacketResamplerCalculator when both Jitter and reflection are
+// enabled.
+class ReproducibleJitterWithReflectionStrategy
+    : public PacketResamplerStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategy(
+      PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ protected:
+  // Returns next random in range (0,n].
+  //
+  // Exposed as virtual function for testing Jitter with reflection.
+  // This is the only way random_ is accessed.
+  virtual uint64 GetNextRandom(uint64 n) {
+    return random_->UnbiasedUniform64(n);
+  }
+
+ private:
+  // Initializes Jitter with reflection.
+  //
+  // This will fast-forward to the period containing current_timestamp.
+  // next_output_timestamp_ is guarnateed to be current_timestamp's period
+  // and packet_emitted_this_period_ will be set to false.
+  void InitializeNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Potentially advances next_output_timestamp_ a single period.
+  //
+  // next_output_timestamp_ will only be advanced if packet_emitted_this_period_
+  // is false.  next_output_timestamp_ will never be advanced beyond
+  // current_timestamp's period.
+  //
+  // However, next_output_timestamp_ could fall before current_timestamp's
+  // period since only a single period can be advanced at a time.
+  void UpdateNextOutputTimestamp(Timestamp current_timestamp);
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // Indicates packet was emitted for current period (i.e. the period
+  // next_output_timestamp_ falls in.
+  bool packet_emitted_this_period_ = false;
+};
+
+// Strategy that applies Jitter without reflection based sampling.
+//
+// Used by PacketResamplerCalculator when Jitter is enabled and reflection is
+// not enabled.
+class JitterWithoutReflectionStrategy : public PacketResamplerStrategy {
+ public:
+  JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestamp();
+
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestamp();
+
+  // Jitter-related variables.
+  std::unique_ptr<RandomBase> random_;
+
+  // Next packet to be emitted.  Since packets may not align perfectly with
+  // next_output_timestamp_, the closest packet will be emitted.
+  Timestamp next_output_timestamp_;
+
+  // Lower bound for next timestamp.
+  //
+  // next_output_timestamp_ will be kept within the interval
+  // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_)
+  Timestamp next_output_timestamp_min_ = Timestamp::Unset();
+
+  // packet reservior used for sampling random packet out of partial period.
+  std::unique_ptr<PacketReservoir> packet_reservoir_;
+
+  // random number generator used in packet_reservior_.
+  std::unique_ptr<RandomBase> packet_reservoir_random_;
+};
+
+// Strategy that applies sampling without any jitter.
+//
+// Used by PacketResamplerCalculator when jitter is not enabled.
+class NoJitterStrategy : public PacketResamplerStrategy {
+ public:
+  NoJitterStrategy(PacketResamplerCalculator* calculator)
+      : PacketResamplerStrategy(calculator) {}
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Number of periods that have passed (= #packets sent to the output).
+  int64 period_count_;
+
+  // If specified, output timestamps are aligned with base_timestamp.
+  // Otherwise, they are aligned with the first input timestamp.
+  Timestamp base_timestamp_;
+};
+
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -68,8 +68,23 @@ message PacketResamplerCalculatorOptions {
   // pseudo-random number generator does its job and the number of frames is
   // sufficiently large, the average frame rate will be close to this value.
   optional double jitter = 4;
+
+  // Enables reflection when applying jitter.
+  //
+  // This option is ignored when reproducible_sampling is true, in which case
+  // reflection will be used.
+  //
+  // New use cases should use reproducible_sampling = true, as
+  // jitter_with_reflection is deprecated and will be removed at some point.
   optional bool jitter_with_reflection = 9 [default = false];
 
+  // If set, enabled reproducible sampling, allowing frames to be sampled
+  // without regards to where the stream starts.  See
+  // packet_resampler_calculator.h for details.
+  //
+  // This enables reflection (ignoring jitter_with_reflection setting).
+  optional bool reproducible_sampling = 10 [default = false];
+
   // If specified, output timestamps are aligned with base_timestamp.
   // Otherwise, they are aligned with the first input timestamp.
   //

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -30,6 +30,7 @@
 
 namespace mediapipe {
 
+using ::testing::ElementsAre;
 namespace {
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
@@ -96,6 +97,77 @@ class SimpleRunner : public CalculatorRunner {
   static int static_count_;
 };
 
+// Matcher for Packets with uint64 payload, comparing arg packet's
+// timestamp and uint64 payload.
+MATCHER_P2(PacketAtTimestamp, payload, timestamp,
+           absl::StrCat(negation ? "isn't" : "is", " a packet with payload ",
+                        payload, " @ time ", timestamp)) {
+  if (timestamp != arg.Timestamp().Value()) {
+    *result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
+    return false;
+  }
+  int64 actual_payload = arg.template Get<int64>();
+  if (actual_payload != payload) {
+    *result_listener << "with incorrect payload = " << actual_payload;
+    return false;
+  }
+  return true;
+}
+
+// JitterWithReflectionStrategy child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleJitterWithReflectionStrategyForTesting
+    : public ReproducibleJitterWithReflectionStrategy {
+ public:
+  ReproducibleJitterWithReflectionStrategyForTesting(
+      PacketResamplerCalculator* calculator)
+      : ReproducibleJitterWithReflectionStrategy(calculator) {}
+
+  // Statically accessed random sequence to use for jitter with reflection.
+  //
+  // An EXPECT will fail if sequence is less than the number requested during
+  // processing.
+  static std::vector<uint64> random_sequence;
+
+ protected:
+  virtual uint64 GetNextRandom(uint64 n) {
+    EXPECT_LT(sequence_index_, random_sequence.size());
+    return random_sequence[sequence_index_++] % n;
+  }
+
+ private:
+  int32 sequence_index_ = 0;
+};
+std::vector<uint64>
+    ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
+
+// PacketResamplerCalculator child class which injects a specified stream
+// of "random" numbers.
+//
+// Calculators are created through factory methods, making testing and injection
+// tricky.  This class utilizes a static variable, random_sequence, to pass
+// the desired random sequence into the calculator.
+class ReproducibleResamplerCalculatorForTesting
+    : public PacketResamplerCalculator {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    return PacketResamplerCalculator::GetContract(cc);
+  }
+
+ protected:
+  std::unique_ptr<class PacketResamplerStrategy> GetSamplingStrategy(
+      const mediapipe::PacketResamplerCalculatorOptions& Options) {
+    return absl::make_unique<
+        ReproducibleJitterWithReflectionStrategyForTesting>(this);
+  }
+};
+
+REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting);
+
 int SimpleRunner::static_count_ = 0;
 
 TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -90,7 +90,7 @@ class PacketThinnerCalculator : public CalculatorBase {
   PacketThinnerCalculator() {}
   ~PacketThinnerCalculator() override {}
 
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     if (cc->InputSidePackets().HasTag(kOptionsTag)) {
       cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
     }
@@ -99,21 +99,21 @@ class PacketThinnerCalculator : public CalculatorBase {
     if (cc->InputSidePackets().HasTag(kPeriodTag)) {
       cc->InputSidePackets().Tag(kPeriodTag).Set<int64>();
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) override;
-  mediapipe::Status Close(CalculatorContext* cc) override;
-  mediapipe::Status Process(CalculatorContext* cc) override {
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override {
     if (cc->InputTimestamp() < start_time_) {
-      return mediapipe::OkStatus();  // Drop packets before start_time_.
+      return absl::OkStatus();  // Drop packets before start_time_.
     } else if (cc->InputTimestamp() >= end_time_) {
       if (!cc->Outputs().Index(0).IsClosed()) {
         cc->Outputs()
             .Index(0)
             .Close();  // No more Packets will be output after end_time_.
       }
-      return mediapipe::OkStatus();
+      return absl::OkStatus();
     } else {
       return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC
                  ? AsyncThinnerProcess(cc)
@@ -123,8 +123,8 @@ class PacketThinnerCalculator : public CalculatorBase {
 
  private:
   // Implementation of ASYNC and SYNC versions of thinner algorithm.
-  mediapipe::Status AsyncThinnerProcess(CalculatorContext* cc);
-  mediapipe::Status SyncThinnerProcess(CalculatorContext* cc);
+  absl::Status AsyncThinnerProcess(CalculatorContext* cc);
+  absl::Status SyncThinnerProcess(CalculatorContext* cc);
 
   // Cached option.
   PacketThinnerCalculatorOptions::ThinnerType thinner_type_;
@@ -153,7 +153,7 @@ namespace {
 TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
 }  // namespace
 
-mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
+absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
   PacketThinnerCalculatorOptions options = mediapipe::tool::RetrieveOptions(
       cc->Options<PacketThinnerCalculatorOptions>(), cc->InputSidePackets(),
       kOptionsTag);
@@ -224,10 +224,10 @@ mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
     }
   }
 
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
+absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
   // Emit any saved packets before quitting.
   if (!saved_packet_.IsEmpty()) {
     // Only sync thinner should have saved packets.
@@ -239,10 +239,10 @@ mediapipe::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
       cc->Outputs().Index(0).AddPacket(saved_packet_);
     }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
+absl::Status PacketThinnerCalculator::AsyncThinnerProcess(
     CalculatorContext* cc) {
   if (cc->InputTimestamp() >= next_valid_timestamp_) {
     cc->Outputs().Index(0).AddPacket(
@@ -251,10 +251,10 @@ mediapipe::Status PacketThinnerCalculator::AsyncThinnerProcess(
     // Guaranteed not to emit packets seen during refractory period.
     cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_);
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
-mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
+absl::Status PacketThinnerCalculator::SyncThinnerProcess(
     CalculatorContext* cc) {
   if (saved_packet_.IsEmpty()) {
     // If no packet has been saved, store the current packet.
@@ -290,7 +290,7 @@ mediapipe::Status PacketThinnerCalculator::SyncThinnerProcess(
       saved_packet_ = cc->Inputs().Index(0).Value();
     }
   }
-  return mediapipe::OkStatus();
+  return absl::OkStatus();
 }
 
 Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {

mediapipe/calculators/core/pass_through_calculator.cc

@@ -28,9 +28,9 @@ namespace mediapipe {
 // ignored.
 class PassThroughCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     if (!cc->Inputs().TagMap()->SameAs(*cc->Outputs().TagMap())) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           "Input and output streams to PassThroughCalculator must use "
           "matching tags and indexes.");
     }
@@ -46,7 +46,7 @@ class PassThroughCalculator : public CalculatorBase {
     if (cc->OutputSidePackets().NumEntries() != 0) {
       if (!cc->InputSidePackets().TagMap()->SameAs(
               *cc->OutputSidePackets().TagMap())) {
-        return mediapipe::InvalidArgumentError(
+        return absl::InvalidArgumentError(
             "Input and output side packets to PassThroughCalculator must use "
             "matching tags and indexes.");
       }
@@ -56,10 +56,10 @@ class PassThroughCalculator : public CalculatorBase {
             &cc->InputSidePackets().Get(id));
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     for (CollectionItemId id = cc->Inputs().BeginId();
          id < cc->Inputs().EndId(); ++id) {
       if (!cc->Inputs().Get(id).Header().IsEmpty()) {
@@ -73,10 +73,10 @@ class PassThroughCalculator : public CalculatorBase {
       }
     }
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     cc->GetCounter("PassThrough")->Increment();
     if (cc->Inputs().NumEntries() == 0) {
       return tool::StatusStop();
@@ -90,7 +90,7 @@ class PassThroughCalculator : public CalculatorBase {
         cc->Outputs().Get(id).AddPacket(cc->Inputs().Get(id).Value());
       }
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 REGISTER_CALCULATOR(PassThroughCalculator);

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -14,12 +14,14 @@
 
 #include <deque>
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // PreviousLoopbackCalculator is useful when a graph needs to process an input
 // together with some previous output.
@@ -51,79 +53,77 @@ namespace mediapipe {
 //   input_stream: "PREV_TRACK:prev_output"
 //   output_stream: "TRACK:output"
 // }
-class PreviousLoopbackCalculator : public CalculatorBase {
+class PreviousLoopbackCalculator : public Node {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Get("MAIN", 0).SetAny();
-    cc->Inputs().Get("LOOP", 0).SetAny();
-    cc->Outputs().Get("PREV_LOOP", 0).SetSameAs(&(cc->Inputs().Get("LOOP", 0)));
-    // TODO: an optional PREV_TIMESTAMP output could be added to
-    // carry the original timestamp of the packet on PREV_LOOP.
-    cc->SetInputStreamHandler("ImmediateInputStreamHandler");
+  static constexpr Input<AnyType> kMain{"MAIN"};
+  static constexpr Input<AnyType> kLoop{"LOOP"};
+  static constexpr Output<SameType<kLoop>> kPrevLoop{"PREV_LOOP"};
+  // TODO: an optional PREV_TIMESTAMP output could be added to
+  // carry the original timestamp of the packet on PREV_LOOP.
+
+  MEDIAPIPE_NODE_CONTRACT(kMain, kLoop, kPrevLoop,
+                          StreamHandler("ImmediateInputStreamHandler"),
+                          TimestampChange::Arbitrary());
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
     // Process() function is invoked in response to MAIN/LOOP stream timestamp
     // bound updates.
     cc->SetProcessTimestampBounds(true);
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
-    main_id_ = cc->Inputs().GetId("MAIN", 0);
-    loop_id_ = cc->Inputs().GetId("LOOP", 0);
-    prev_loop_id_ = cc->Outputs().GetId("PREV_LOOP", 0);
-    cc->Outputs()
-        .Get(prev_loop_id_)
-        .SetHeader(cc->Inputs().Get(loop_id_).Header());
-    return mediapipe::OkStatus();
+  absl::Status Open(CalculatorContext* cc) final {
+    kPrevLoop(cc).SetHeader(kLoop(cc).Header());
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     // Non-empty packets and empty packets indicating timestamp bound updates
     // are guaranteed to have timestamps greater than timestamps of previous
     // packets within the same stream. Calculator tracks and operates on such
     // packets.
 
-    const Packet& main_packet = cc->Inputs().Get(main_id_).Value();
-    if (prev_main_ts_ < main_packet.Timestamp()) {
+    const PacketBase& main_packet = kMain(cc).packet();
+    if (prev_main_ts_ < main_packet.timestamp()) {
       Timestamp loop_timestamp;
       if (!main_packet.IsEmpty()) {
         loop_timestamp = prev_non_empty_main_ts_;
-        prev_non_empty_main_ts_ = main_packet.Timestamp();
+        prev_non_empty_main_ts_ = main_packet.timestamp();
       } else {
         // Calculator advances PREV_LOOP timestamp bound in response to empty
         // MAIN packet, hence not caring about corresponding loop packet.
         loop_timestamp = Timestamp::Unset();
       }
-      main_packet_specs_.push_back({main_packet.Timestamp(), loop_timestamp});
-      prev_main_ts_ = main_packet.Timestamp();
+      main_packet_specs_.push_back({main_packet.timestamp(), loop_timestamp});
+      prev_main_ts_ = main_packet.timestamp();
     }
 
-    const Packet& loop_packet = cc->Inputs().Get(loop_id_).Value();
-    if (prev_loop_ts_ < loop_packet.Timestamp()) {
+    const PacketBase& loop_packet = kLoop(cc).packet();
+    if (prev_loop_ts_ < loop_packet.timestamp()) {
       loop_packets_.push_back(loop_packet);
-      prev_loop_ts_ = loop_packet.Timestamp();
+      prev_loop_ts_ = loop_packet.timestamp();
     }
 
-    auto& prev_loop = cc->Outputs().Get(prev_loop_id_);
     while (!main_packet_specs_.empty() && !loop_packets_.empty()) {
       // The earliest MAIN packet.
-      const MainPacketSpec& main_spec = main_packet_specs_.front();
+      MainPacketSpec main_spec = main_packet_specs_.front();
       // The earliest LOOP packet.
-      const Packet& loop_candidate = loop_packets_.front();
+      const PacketBase& loop_candidate = loop_packets_.front();
       // Match LOOP and MAIN packets.
-      if (main_spec.loop_timestamp < loop_candidate.Timestamp()) {
+      if (main_spec.loop_timestamp < loop_candidate.timestamp()) {
         // No LOOP packet can match the MAIN packet under review.
-        prev_loop.SetNextTimestampBound(main_spec.timestamp + 1);
+        kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
         main_packet_specs_.pop_front();
-      } else if (main_spec.loop_timestamp > loop_candidate.Timestamp()) {
+      } else if (main_spec.loop_timestamp > loop_candidate.timestamp()) {
         // No MAIN packet can match the LOOP packet under review.
         loop_packets_.pop_front();
       } else {
         // Exact match found.
         if (loop_candidate.IsEmpty()) {
           // However, LOOP packet is empty.
-          prev_loop.SetNextTimestampBound(main_spec.timestamp + 1);
+          kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
         } else {
-          prev_loop.AddPacket(loop_candidate.At(main_spec.timestamp));
+          kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
         }
         loop_packets_.pop_front();
         main_packet_specs_.pop_front();
@@ -135,11 +135,11 @@ class PreviousLoopbackCalculator : public CalculatorBase {
       // b) Empty MAIN packet has been received with Timestamp::Max() indicating
       //    MAIN is done.
       if (main_spec.timestamp == Timestamp::Done().PreviousAllowedInStream()) {
-        prev_loop.Close();
+        kPrevLoop(cc).Close();
       }
     }
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:
@@ -150,10 +150,6 @@ class PreviousLoopbackCalculator : public CalculatorBase {
     Timestamp loop_timestamp;
   };
 
-  CollectionItemId main_id_;
-  CollectionItemId loop_id_;
-  CollectionItemId prev_loop_id_;
-
   // Contains specs for MAIN packets which only can be:
   // - non-empty packets
   // - empty packets indicating timestamp bound updates
@@ -169,12 +165,13 @@ class PreviousLoopbackCalculator : public CalculatorBase {
   // - empty packets indicating timestamp bound updates
   //
   // Sorted according to packet timestamps.
-  std::deque<Packet> loop_packets_;
+  std::deque<PacketBase> loop_packets_;
   // Using "Timestamp::Unset" instead of "Timestamp::Unstarted" in order to
   // allow addition of the very first empty packet (which doesn't indicate
   // timestamp bound change necessarily).
   Timestamp prev_loop_ts_ = Timestamp::Unset();
 };
-REGISTER_CALCULATOR(PreviousLoopbackCalculator);
+MEDIAPIPE_REGISTER_NODE(PreviousLoopbackCalculator);
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -136,27 +136,27 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
 // A Calculator that outputs a summary packet in CalculatorBase::Close().
 class PacketOnCloseCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<int>();
     cc->Outputs().Index(0).Set<int>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     cc->SetOffset(TimestampDiff(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     sum_ += cc->Inputs().Index(0).Value().Get<int>();
     cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Close(CalculatorContext* cc) final {
+  absl::Status Close(CalculatorContext* cc) final {
     cc->Outputs().Index(0).AddPacket(
         MakePacket<int>(sum_).At(Timestamp::Max()));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:
@@ -700,19 +700,19 @@ TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest,
 // Similar to GateCalculator, but it doesn't propagate timestamp bound updates.
 class DroppingGateCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).SetAny();
     cc->Inputs().Tag("DISALLOW").Set<bool>();
     cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     if (!cc->Inputs().Index(0).IsEmpty() &&
         !cc->Inputs().Tag("DISALLOW").Get<bool>()) {
       cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 };
 REGISTER_CALCULATOR(DroppingGateCalculator);

mediapipe/calculators/core/quantize_float_vector_calculator.cc

@@ -43,32 +43,32 @@ namespace mediapipe {
 
 class QuantizeFloatVectorCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
     cc->Outputs().Tag("ENCODED").Set<std::string>();
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     const auto options =
         cc->Options<::mediapipe::QuantizeFloatVectorCalculatorOptions>();
     if (!options.has_max_quantized_value() ||
         !options.has_min_quantized_value()) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           "Both max_quantized_value and min_quantized_value must be provided "
           "in QuantizeFloatVectorCalculatorOptions.");
     }
     max_quantized_value_ = options.max_quantized_value();
     min_quantized_value_ = options.min_quantized_value();
     if (max_quantized_value_ < min_quantized_value_ + FLT_EPSILON) {
-      return mediapipe::InvalidArgumentError(
+      return absl::InvalidArgumentError(
           "max_quantized_value must be greater than min_quantized_value.");
     }
     range_ = max_quantized_value_ - min_quantized_value_;
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     const std::vector<float>& float_vector =
         cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
     int feature_size = float_vector.size();
@@ -88,7 +88,7 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
     }
     cc->Outputs().Tag("ENCODED").AddPacket(
         MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private:

mediapipe/calculators/core/real_time_flow_limiter_calculator.cc

@@ -75,7 +75,7 @@ namespace mediapipe {
 // }
 class RealTimeFlowLimiterCalculator : public CalculatorBase {
  public:
-  static mediapipe::Status GetContract(CalculatorContract* cc) {
+  static absl::Status GetContract(CalculatorContract* cc) {
     int num_data_streams = cc->Inputs().NumEntries("");
     RET_CHECK_GE(num_data_streams, 1);
     RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
@@ -95,10 +95,10 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
 
     cc->SetInputStreamHandler("ImmediateInputStreamHandler");
 
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
-  mediapipe::Status Open(CalculatorContext* cc) final {
+  absl::Status Open(CalculatorContext* cc) final {
     finished_id_ = cc->Inputs().GetId("FINISHED", 0);
     max_in_flight_ = 1;
     if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
@@ -113,12 +113,12 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
     num_data_streams_ = cc->Inputs().NumEntries("");
     data_stream_bound_ts_.resize(num_data_streams_);
     RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
   bool Allow() { return num_in_flight_ < max_in_flight_; }
 
-  mediapipe::Status Process(CalculatorContext* cc) final {
+  absl::Status Process(CalculatorContext* cc) final {
     bool old_allow = Allow();
     Timestamp lowest_incomplete_ts = Timestamp::Done();
 
@@ -180,7 +180,7 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
           .Get(allowed_id_)
           .AddPacket(MakePacket<bool>(Allow()).At(++allow_ctr_ts_));
     }
-    return mediapipe::OkStatus();
+    return absl::OkStatus();
   }
 
  private: