Project import generated by Copybara.

GitOrigin-RevId: f4b1fe3f15810450fb6539e733f6a260d3ee082c

README.md

@@ -45,7 +45,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             | ✅                                                             |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
@@ -79,6 +79,13 @@ run code search using
 
 ## Publications
 
+*   [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
+    in Google Developers Blog
+*   [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html)
+    in Google Developers Blog
+*   [SignAll SDK: Sign language interface using MediaPipe is now available for
+    developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
+    in Google Developers Blog
 *   [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
     in Google AI Blog
 *   [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)

WORKSPACE

@@ -53,19 +53,12 @@ rules_foreign_cc_dependencies()
 all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])"""
 
 # GoogleTest/GoogleMock framework. Used by most unit-tests.
-# Last updated 2020-06-30.
+# Last updated 2021-07-02.
 http_archive(
     name = "com_google_googletest",
-    urls = ["https://github.com/google/googletest/archive/aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e.zip"],
+    urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"],
-    patches = [
+    strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3",
-        # fix for https://github.com/google/googletest/issues/2817
+    sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049",
-        "@//third_party:com_google_googletest_9d580ea80592189e6d44fa35bcf9cdea8bf620d6.diff"
-    ],
-    patch_args = [
-        "-p1",
-    ],
-    strip_prefix = "googletest-aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e",
-    sha256 = "04a1751f94244307cebe695a69cc945f9387a80b0ef1af21394a490697c5c895",
 )
 
 # Google Benchmark library.
@@ -164,11 +157,11 @@ http_archive(
 http_archive(
     name = "pybind11",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
-        "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
+        "https://github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
     ],
-    sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d",
+    sha256 = "616d1c42e4cf14fa27b2a4ff759d7d7b33006fdc5ad8fd603bb2c22622f27020",
-    strip_prefix = "pybind11-2.4.3",
+    strip_prefix = "pybind11-2.7.1",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
@@ -338,7 +331,9 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 maven_install(
     artifacts = [
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
-        "androidx.lifecycle:lifecycle-common:2.2.0",
+        "androidx.lifecycle:lifecycle-common:2.3.1",
+        "androidx.activity:activity:1.2.2",
+        "androidx.fragment:fragment:1.3.4",
         "androidx.annotation:annotation:aar:1.1.0",
         "androidx.appcompat:appcompat:aar:1.1.0-rc01",
         "androidx.camera:camera-core:1.0.0-beta10",
@@ -353,9 +348,9 @@ maven_install(
         "com.google.android.material:material:aar:1.0.0-rc01",
         "com.google.auto.value:auto-value:1.8.1",
         "com.google.auto.value:auto-value-annotations:1.8.1",
-        "com.google.code.findbugs:jsr305:3.0.2",
+        "com.google.code.findbugs:jsr305:latest.release",
-        "com.google.flogger:flogger-system-backend:0.3.1",
+        "com.google.flogger:flogger-system-backend:latest.release",
-        "com.google.flogger:flogger:0.3.1",
+        "com.google.flogger:flogger:latest.release",
         "com.google.guava:guava:27.0.1-android",
         "com.google.guava:listenablefuture:1.0",
         "junit:junit:4.12",
@@ -383,9 +378,9 @@ http_archive(
 )
 
 # Tensorflow repo should always go after the other external dependencies.
-# 2021-06-07
+# 2021-07-29
-_TENSORFLOW_GIT_COMMIT = "700533808e6016dc458bb2eeecfca4babfc482ec"
+_TENSORFLOW_GIT_COMMIT = "52a2905cbc21034766c08041933053178c5d10e3"
-_TENSORFLOW_SHA256 = "b6edd7f4039bfc19f3e77594ecff558ba620091d0dc48181484b3d9085026126"
+_TENSORFLOW_SHA256 = "06d4691bcdb700f3275fa0971a1585221c2b9f3dffe867963be565a6643d7f56"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -406,3 +401,18 @@ load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")
 tf_workspace3()
 load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")
 tf_workspace2()
+
+# Edge TPU
+http_archive(
+  name = "libedgetpu",
+  sha256 = "14d5527a943a25bc648c28a9961f954f70ba4d79c0a9ca5ae226e1831d72fe80",
+  strip_prefix = "libedgetpu-3164995622300286ef2bb14d7fdc2792dae045b7",
+  urls = [
+    "https://github.com/google-coral/libedgetpu/archive/3164995622300286ef2bb14d7fdc2792dae045b7.tar.gz"
+  ],
+)
+load("@libedgetpu//:workspace.bzl", "libedgetpu_dependencies")
+libedgetpu_dependencies()
+
+load("@coral_crosstool//:configure.bzl", "cc_crosstool")
+cc_crosstool(name = "crosstool")

docs/getting_started/android.md

@@ -16,12 +16,14 @@ nav_order: 1
 
 Please follow instructions below to build Android example apps in the supported
 MediaPipe [solutions](../solutions/solutions.md). To learn more about these
-example apps, start from [Hello World! on Android](./hello_world_android.md). To
+example apps, start from [Hello World! on Android](./hello_world_android.md).
-incorporate MediaPipe into an existing Android Studio project, see these
-[instructions](./android_archive_library.md) that use Android Archive (AAR) and
-Gradle.
 
-## Building Android example apps
+To incorporate MediaPipe into Android Studio projects, see these
+[instructions](./android_solutions.md) to use the MediaPipe Android Solution
+APIs (currently in alpha) that are now available in
+[Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
+
+## Building Android example apps with Bazel
 
 ### Prerequisite
 
@@ -51,16 +53,6 @@ $YOUR_INTENDED_API_LEVEL` in android_ndk_repository() and/or
 android_sdk_repository() in the
 [`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE) file.
 
-Please verify all the necessary packages are installed.
-
-*   Android SDK Platform API Level 28 or 29
-*   Android SDK Build-Tools 28 or 29
-*   Android SDK Platform-Tools 28 or 29
-*   Android SDK Tools 26.1.1
-*   Android NDK 19c or above
-
-### Option 1: Build with Bazel in Command Line
-
 Tip: You can run this
 [script](https://github.com/google/mediapipe/blob/master/build_android_examples.sh)
 to build (and install) all MediaPipe Android example apps.
@@ -84,108 +76,3 @@ to build (and install) all MediaPipe Android example apps.
     ```bash
     adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/handtrackinggpu.apk
     ```
-
-### Option 2: Build with Bazel in Android Studio
-
-The MediaPipe project can be imported into Android Studio using the Bazel
-plugins. This allows the MediaPipe examples to be built and modified in Android
-Studio.
-
-To incorporate MediaPipe into an existing Android Studio project, see these
-[instructions](./android_archive_library.md) that use Android Archive (AAR) and
-Gradle.
-
-The steps below use Android Studio 3.5 to build and install a MediaPipe example
-app:
-
-1.  Install and launch Android Studio 3.5.
-
-2.  Select `Configure` -> `SDK Manager` -> `SDK Platforms`.
-
-    *   Verify that Android SDK Platform API Level 28 or 29 is installed.
-    *   Take note of the Android SDK Location, e.g.,
-        `/usr/local/home/Android/Sdk`.
-
-3.  Select `Configure` -> `SDK Manager` -> `SDK Tools`.
-
-    *   Verify that Android SDK Build-Tools 28 or 29 is installed.
-    *   Verify that Android SDK Platform-Tools 28 or 29 is installed.
-    *   Verify that Android SDK Tools 26.1.1 is installed.
-    *   Verify that Android NDK 19c or above is installed.
-    *   Take note of the Android NDK Location, e.g.,
-        `/usr/local/home/Android/Sdk/ndk-bundle` or
-        `/usr/local/home/Android/Sdk/ndk/20.0.5594570`.
-
-4.  Set environment variables `$ANDROID_HOME` and `$ANDROID_NDK_HOME` to point
-    to the installed SDK and NDK.
-
-    ```bash
-    export ANDROID_HOME=/usr/local/home/Android/Sdk
-
-    # If the NDK libraries are installed by a previous version of Android Studio, do
-    export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk-bundle
-    # If the NDK libraries are installed by Android Studio 3.5, do
-    export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk/<version number>
-    ```
-
-5.  Select `Configure` -> `Plugins` to install `Bazel`.
-
-6.  On Linux, select `File` -> `Settings` -> `Bazel settings`. On macos, select
-    `Android Studio` -> `Preferences` -> `Bazel settings`. Then, modify `Bazel
-    binary location` to be the same as the output of `$ which bazel`.
-
-7.  Select `Import Bazel Project`.
-
-    *   Select `Workspace`: `/path/to/mediapipe` and select `Next`.
-    *   Select `Generate from BUILD file`: `/path/to/mediapipe/BUILD` and select
-        `Next`.
-    *   Modify `Project View` to be the following and select `Finish`.
-
-    ```
-    directories:
-      # read project settings, e.g., .bazelrc
-      .
-      -mediapipe/objc
-      -mediapipe/examples/ios
-
-    targets:
-      //mediapipe/examples/android/...:all
-      //mediapipe/java/...:all
-
-    android_sdk_platform: android-29
-
-    sync_flags:
-      --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
-    ```
-
-8.  Select `Bazel` -> `Sync` -> `Sync project with Build files`.
-
-    Note: Even after doing step 4, if you still see the error: `"no such package
-    '@androidsdk//': Either the path attribute of android_sdk_repository or the
-    ANDROID_HOME environment variable must be set."`, please modify the
-    [`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE)
-    file to point to your SDK and NDK library locations, as below:
-
-    ```
-    android_sdk_repository(
-        name = "androidsdk",
-        path = "/path/to/android/sdk"
-    )
-
-    android_ndk_repository(
-        name = "androidndk",
-        path = "/path/to/android/ndk"
-    )
-    ```
-
-9.  Connect an Android device to the workstation.
-
-10. Select `Run...` -> `Edit Configurations...`.
-
-    *   Select `Templates` -> `Bazel Command`.
-    *   Enter Target Expression:
-        `//mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu:handtrackinggpu`
-    *   Enter Bazel command: `mobile-install`.
-    *   Enter Bazel flags: `-c opt --config=android_arm64`.
-    *   Press the `[+]` button to add the new configuration.
-    *   Select `Run` to run the example app on the connected Android device.

docs/getting_started/android_archive_library.md

@@ -3,7 +3,7 @@ layout: default
 title: MediaPipe Android Archive
 parent: MediaPipe on Android
 grand_parent: Getting Started
-nav_order: 2
+nav_order: 3
 ---
 
 # MediaPipe Android Archive
@@ -113,9 +113,9 @@ each project.
         androidTestImplementation 'androidx.test.ext:junit:1.1.0'
         androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
         // MediaPipe deps
-        implementation 'com.google.flogger:flogger:0.3.1'
+        implementation 'com.google.flogger:flogger:latest.release'
-        implementation 'com.google.flogger:flogger-system-backend:0.3.1'
+        implementation 'com.google.flogger:flogger-system-backend:latest.release'
-        implementation 'com.google.code.findbugs:jsr305:3.0.2'
+        implementation 'com.google.code.findbugs:jsr305:latest.release'
         implementation 'com.google.guava:guava:27.0.1-android'
         implementation 'com.google.protobuf:protobuf-java:3.11.4'
         // CameraX core library

docs/getting_started/android_solutions.md

@@ -0,0 +1,79 @@
+---
+layout: default
+title: Android Solutions
+parent: MediaPipe on Android
+grand_parent: Getting Started
+nav_order: 2
+---
+
+# Android Solution APIs
+{: .no_toc }
+
+1. TOC
+{:toc}
+---
+
+Please follow instructions below to use the MediaPipe Solution APIs in Android
+Studio projects and build the Android example apps in the supported MediaPipe
+[solutions](../solutions/solutions.md).
+
+## Integrate MediaPipe Android Solutions in Android Studio
+
+MediaPipe Android Solution APIs (currently in alpha) are now available in
+[Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
+To incorporate MediaPipe Android Solutions into an Android Studio project, add
+the following into the project's Gradle dependencies:
+
+```
+dependencies {
+    // MediaPipe solution-core is the foundation of any MediaPipe solutions.
+    implementation 'com.google.mediapipe:solution-core:latest.release'
+    // Optional: MediaPipe Hands solution.
+    implementation 'com.google.mediapipe:hands:latest.release'
+    // Optional: MediaPipe FaceMesh solution.
+    implementation 'com.google.mediapipe:facemesh:latest.release'
+    // MediaPipe deps
+    implementation 'com.google.flogger:flogger:latest.release'
+    implementation 'com.google.flogger:flogger-system-backend:latest.release'
+    implementation 'com.google.guava:guava:27.0.1-android'
+    implementation 'com.google.protobuf:protobuf-java:3.11.4'
+    // CameraX core library
+    def camerax_version = "1.0.0-beta10"
+    implementation "androidx.camera:camera-core:$camerax_version"
+    implementation "androidx.camera:camera-camera2:$camerax_version"
+    implementation "androidx.camera:camera-lifecycle:$camerax_version"
+}
+```
+
+See the detailed solutions API usage examples for different use cases in the
+solution example apps'
+[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
+If the prebuilt maven packages are not sufficient, building the MediaPipe
+Android archive library locally by following these
+[instructions](./android_archive_library.md).
+
+## Build solution example apps in Android Studio
+
+1.  Open Android Studio Arctic Fox on Linux, macOS, or Windows.
+
+2.  Import mediapipe/examples/android/solutions directory into Android Studio.
+
+    ![Screenshot](../images/import_mp_android_studio_project.png)
+
+3.  For Windows users, run `create_win_symlinks.bat` as administrator to create
+    res directory symlinks.
+
+    ![Screenshot](../images/run_create_win_symlinks.png)
+
+4.  Select "File" -> "Sync Project with Gradle Files" to sync project.
+
+5.  Run solution example app in Android Studio.
+
+    ![Screenshot](../images/run_android_solution_app.png)
+
+6.  (Optional) Run solutions on CPU.
+
+    MediaPipe solution example apps run the pipeline and the model inference on
+    GPU by default. If needed, for example to run the apps on Android Emulator,
+    set the `RUN_ON_GPU` boolean variable to `false` in the app's
+    MainActivity.java to run the pipeline and the model inference on CPU.

docs/getting_started/hello_world_android.md

@@ -31,8 +31,8 @@ stream on an Android device.
 
 ## Setup
 
-1.  Install MediaPipe on your system, see [MediaPipe installation guide] for
+1.  Install MediaPipe on your system, see
-    details.
+    [MediaPipe installation guide](./install.md) for details.
 2.  Install Android Development SDK and Android NDK. See how to do so also in
     [MediaPipe installation guide].
 3.  Enable [developer options] on your Android device.
@@ -770,7 +770,6 @@ If you ran into any issues, please see the full code of the tutorial
 [`ExternalTextureConverter`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
 [`FrameLayout`]:https://developer.android.com/reference/android/widget/FrameLayout
 [`FrameProcessor`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java
-[MediaPipe installation guide]:./install.md
 [`PermissionHelper`]: https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/PermissionHelper.java
 [`SurfaceHolder.Callback`]:https://developer.android.com/reference/android/view/SurfaceHolder.Callback.html
 [`SurfaceView`]:https://developer.android.com/reference/android/view/SurfaceView

docs/getting_started/hello_world_ios.md

@@ -31,8 +31,8 @@ stream on an iOS device.
 
 ## Setup
 
-1.  Install MediaPipe on your system, see [MediaPipe installation guide] for
+1.  Install MediaPipe on your system, see
-    details.
+    [MediaPipe installation guide](./install.md) for details.
 2.  Setup your iOS device for development.
 3.  Setup [Bazel] on your system to build and deploy the iOS app.
 
@@ -113,6 +113,10 @@ bazel to build the iOS application. The content of the
 5.  `Main.storyboard` and `Launch.storyboard`
 6.  `Assets.xcassets` directory.
 
+Note: In newer versions of Xcode, you may see additional files `SceneDelegate.h`
+and `SceneDelegate.m`. Make sure to copy them too and add them to the `BUILD`
+file mentioned below.
+
 Copy these files to a directory named `HelloWorld` to a location that can access
 the MediaPipe source code. For example, the source code of the application that
 we will build in this tutorial is located in
@@ -247,6 +251,12 @@ We need to get frames from the `_cameraSource` into our application
 `MPPInputSourceDelegate`. So our application `ViewController` can be a delegate
 of `_cameraSource`.
 
+Update the interface definition of `ViewController` accordingly:
+
+```
+@interface ViewController () <MPPInputSourceDelegate>
+```
+
 To handle camera setup and process incoming frames, we should use a queue
 different from the main queue. Add the following to the implementation block of
 the `ViewController`:
@@ -288,6 +298,12 @@ utility called `MPPLayerRenderer` to display images on the screen. This utility
 can be used to display `CVPixelBufferRef` objects, which is the type of the
 images provided by `MPPCameraInputSource` to its delegates.
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPLayerRenderer.h"
+```
+
 To display images of the screen, we need to add a new `UIView` object called
 `_liveView` to the `ViewController`.
 
@@ -411,6 +427,12 @@ Objective-C++.
 
 ### Use the graph in `ViewController`
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPGraph.h"
+```
+
 Declare a static constant with the name of the graph, the input stream and the
 output stream:
 
@@ -549,6 +571,12 @@ method to receive packets on this output stream and display them on the screen:
 }
 ```
 
+Update the interface definition of `ViewController` with `MPPGraphDelegate`:
+
+```
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+```
+
 And that is all! Build and run the app on your iOS device. You should see the
 results of running the edge detection graph on a live video feed. Congrats!
 
@@ -560,6 +588,5 @@ appropriate `BUILD` file dependencies for the edge detection graph.
 
 [Bazel]:https://bazel.build/
 [`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
-[MediaPipe installation guide]:./install.md
+[common]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common
-[common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common)
+[helloworld]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld
-[helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld)

docs/getting_started/install.md

@@ -43,104 +43,189 @@ install --user six`.
 
 3.  Install OpenCV and FFmpeg.
 
-    Option 1. Use package manager tool to install the pre-compiled OpenCV
+    **Option 1**. Use package manager tool to install the pre-compiled OpenCV
-    libraries. FFmpeg will be installed via libopencv-video-dev.
+    libraries. FFmpeg will be installed via `libopencv-video-dev`.
 
-    Note: Debian 9 and Ubuntu 16.04 provide OpenCV 2.4.9. You may want to take
+    OS                   | OpenCV
-    option 2 or 3 to install OpenCV 3 or above.
+    -------------------- | ------
+    Debian 9 (stretch)   | 2.4
+    Debian 10 (buster)   | 3.2
+    Debian 11 (bullseye) | 4.5
+    Ubuntu 16.04 LTS     | 2.4
+    Ubuntu 18.04 LTS     | 3.2
+    Ubuntu 20.04 LTS     | 4.2
+    Ubuntu 20.04 LTS     | 4.2
+    Ubuntu 21.04         | 4.5
 
     ```bash
-    $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+    $ sudo apt-get install -y \
-                           libopencv-calib3d-dev libopencv-features2d-dev \
+        libopencv-core-dev \
-                           libopencv-imgproc-dev libopencv-video-dev
+        libopencv-highgui-dev \
+        libopencv-calib3d-dev \
+        libopencv-features2d-dev \
+        libopencv-imgproc-dev \
+        libopencv-video-dev
     ```
 
-    Debian 9 and Ubuntu 18.04 install the packages in
+    MediaPipe's [`opencv_linux.BUILD`] and [`WORKSPACE`] are already configured
-    `/usr/lib/x86_64-linux-gnu`. MediaPipe's [`opencv_linux.BUILD`] and
+    for OpenCV 2/3 and should work correctly on any architecture:
-    [`ffmpeg_linux.BUILD`] are configured for this library path. Ubuntu 20.04
-    may install the OpenCV and FFmpeg packages in `/usr/local`, Please follow
-    the option 3 below to modify the [`WORKSPACE`], [`opencv_linux.BUILD`] and
-    [`ffmpeg_linux.BUILD`] files accordingly.
-
-    Moreover, for Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, the
-    library path needs to be modified like the following:
 
     ```bash
-    sed -i "s/x86_64-linux-gnu/aarch64-linux-gnu/g" third_party/opencv_linux.BUILD
+    # WORKSPACE
+    new_local_repository(
+      name = "linux_opencv",
+      build_file = "@//third_party:opencv_linux.BUILD",
+      path = "/usr",
+    )
+
+    # opencv_linux.BUILD for OpenCV 2/3 installed from Debian package
+    cc_library(
+      name = "opencv",
+      linkopts = [
+        "-l:libopencv_core.so",
+        "-l:libopencv_calib3d.so",
+        "-l:libopencv_features2d.so",
+        "-l:libopencv_highgui.so",
+        "-l:libopencv_imgcodecs.so",
+        "-l:libopencv_imgproc.so",
+        "-l:libopencv_video.so",
+        "-l:libopencv_videoio.so",
+      ],
+    )
     ```
 
-    Option 2. Run [`setup_opencv.sh`] to automatically build OpenCV from source
+    For OpenCV 4 you need to modify [`opencv_linux.BUILD`] taking into account
-    and modify MediaPipe's OpenCV config.
+    current architecture:
 
-    Option 3. Follow OpenCV's
+    ```bash
+    # WORKSPACE
+    new_local_repository(
+      name = "linux_opencv",
+      build_file = "@//third_party:opencv_linux.BUILD",
+      path = "/usr",
+    )
+
+    # opencv_linux.BUILD for OpenCV 4 installed from Debian package
+    cc_library(
+      name = "opencv",
+      hdrs = glob([
+        # Uncomment according to your multiarch value (gcc -print-multiarch):
+        #  "include/aarch64-linux-gnu/opencv4/opencv2/cvconfig.h",
+        #  "include/arm-linux-gnueabihf/opencv4/opencv2/cvconfig.h",
+        #  "include/x86_64-linux-gnu/opencv4/opencv2/cvconfig.h",
+        "include/opencv4/opencv2/**/*.h*",
+      ]),
+      includes = [
+        # Uncomment according to your multiarch value (gcc -print-multiarch):
+        #  "include/aarch64-linux-gnu/opencv4/",
+        #  "include/arm-linux-gnueabihf/opencv4/",
+        #  "include/x86_64-linux-gnu/opencv4/",
+        "include/opencv4/",
+      ],
+      linkopts = [
+        "-l:libopencv_core.so",
+        "-l:libopencv_calib3d.so",
+        "-l:libopencv_features2d.so",
+        "-l:libopencv_highgui.so",
+        "-l:libopencv_imgcodecs.so",
+        "-l:libopencv_imgproc.so",
+        "-l:libopencv_video.so",
+        "-l:libopencv_videoio.so",
+      ],
+    )
+    ```
+
+    **Option 2**. Run [`setup_opencv.sh`] to automatically build OpenCV from
+    source and modify MediaPipe's OpenCV config. This option will do all steps
+    defined in Option 3 automatically.
+
+    **Option 3**. Follow OpenCV's
     [documentation](https://docs.opencv.org/3.4.6/d7/d9f/tutorial_linux_install.html)
     to manually build OpenCV from source code.
 
-    Note: You may need to modify [`WORKSPACE`], [`opencv_linux.BUILD`] and
+    You may need to modify [`WORKSPACE`] and [`opencv_linux.BUILD`] to point
-    [`ffmpeg_linux.BUILD`] to point MediaPipe to your own OpenCV and FFmpeg
+    MediaPipe to your own OpenCV libraries. Assume OpenCV would be installed to
-    libraries. For example if OpenCV and FFmpeg are both manually installed in
+    `/usr/local/` which is recommended by default.
-    "/usr/local/", you will need to update: (1) the "linux_opencv" and
+
-    "linux_ffmpeg" new_local_repository rules in [`WORKSPACE`], (2) the "opencv"
+    OpenCV 2/3 setup:
-    cc_library rule in [`opencv_linux.BUILD`], and (3) the "libffmpeg"
-    cc_library rule in [`ffmpeg_linux.BUILD`]. These 3 changes are shown below:
 
     ```bash
+    # WORKSPACE
     new_local_repository(
-        name = "linux_opencv",
+      name = "linux_opencv",
-        build_file = "@//third_party:opencv_linux.BUILD",
+      build_file = "@//third_party:opencv_linux.BUILD",
-        path = "/usr/local",
+      path = "/usr/local",
     )
 
+    # opencv_linux.BUILD for OpenCV 2/3 installed to /usr/local
+    cc_library(
+      name = "opencv",
+      linkopts = [
+        "-L/usr/local/lib",
+        "-l:libopencv_core.so",
+        "-l:libopencv_calib3d.so",
+        "-l:libopencv_features2d.so",
+        "-l:libopencv_highgui.so",
+        "-l:libopencv_imgcodecs.so",
+        "-l:libopencv_imgproc.so",
+        "-l:libopencv_video.so",
+        "-l:libopencv_videoio.so",
+      ],
+    )
+    ```
+
+    OpenCV 4 setup:
+
+    ```bash
+    # WORKSPACE
     new_local_repository(
-        name = "linux_ffmpeg",
+      name = "linux_opencv",
-        build_file = "@//third_party:ffmpeg_linux.BUILD",
+      build_file = "@//third_party:opencv_linux.BUILD",
-        path = "/usr/local",
+      path = "/usr/local",
     )
 
+    # opencv_linux.BUILD for OpenCV 4 installed to /usr/local
     cc_library(
-        name = "opencv",
+      name = "opencv",
-        srcs = glob(
+      hdrs = glob([
-            [
+        "include/opencv4/opencv2/**/*.h*",
-                "lib/libopencv_core.so",
+      ]),
-                "lib/libopencv_highgui.so",
+      includes = [
-                "lib/libopencv_imgcodecs.so",
+        "include/opencv4/",
-                "lib/libopencv_imgproc.so",
+      ],
-                "lib/libopencv_video.so",
+      linkopts = [
-                "lib/libopencv_videoio.so",
+        "-L/usr/local/lib",
-            ],
+        "-l:libopencv_core.so",
-        ),
+        "-l:libopencv_calib3d.so",
-        hdrs = glob([
+        "-l:libopencv_features2d.so",
-            # For OpenCV 3.x
+        "-l:libopencv_highgui.so",
-            "include/opencv2/**/*.h*",
+        "-l:libopencv_imgcodecs.so",
-            # For OpenCV 4.x
+        "-l:libopencv_imgproc.so",
-            # "include/opencv4/opencv2/**/*.h*",
+        "-l:libopencv_video.so",
-        ]),
+        "-l:libopencv_videoio.so",
-        includes = [
+      ],
-            # For OpenCV 3.x
+    )
-            "include/",
+    ```
-            # For OpenCV 4.x
+
-            # "include/opencv4/",
+    Current FFmpeg setup is defined in [`ffmpeg_linux.BUILD`] and should work
-        ],
+    for any architecture:
-        linkstatic = 1,
+
-        visibility = ["//visibility:public"],
+    ```bash
+    # WORKSPACE
+    new_local_repository(
+      name = "linux_ffmpeg",
+      build_file = "@//third_party:ffmpeg_linux.BUILD",
+      path = "/usr"
     )
 
+    # ffmpeg_linux.BUILD for FFmpeg installed from Debian package
     cc_library(
-        name = "libffmpeg",
+      name = "libffmpeg",
-        srcs = glob(
+      linkopts = [
-            [
+        "-l:libavcodec.so",
-                "lib/libav*.so",
+        "-l:libavformat.so",
-            ],
+        "-l:libavutil.so",
-        ),
+      ],
-        hdrs = glob(["include/libav*/*.h"]),
-        includes = ["include"],
-        linkopts = [
-            "-lavcodec",
-            "-lavformat",
-            "-lavutil",
-        ],
-        linkstatic = 1,
-        visibility = ["//visibility:public"],
     )
     ```
 
@@ -711,7 +796,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
     ```bash
     $ docker run -it --name mediapipe mediapipe:latest
 
-    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
+    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazelisk run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
 
     # Should print:
     # Hello World!

docs/getting_started/javascript.md

@@ -22,12 +22,23 @@ Solution                    | NPM Package                             | Example
 [Face Detection][Fd-pg]     | [@mediapipe/face_detection][Fd-npm]     | [mediapipe.dev/demo/face_detection][Fd-demo]
 [Hands][H-pg]               | [@mediapipe/hands][H-npm]               | [mediapipe.dev/demo/hands][H-demo]
 [Holistic][Ho-pg]           | [@mediapipe/holistic][Ho-npm]           | [mediapipe.dev/demo/holistic][Ho-demo]
+[Objectron][Ob-pg]          | [@mediapipe/objectron][Ob-npm]          | [mediapipe.dev/demo/objectron][Ob-demo]
 [Pose][P-pg]                | [@mediapipe/pose][P-npm]                | [mediapipe.dev/demo/pose][P-demo]
 [Selfie Segmentation][S-pg] | [@mediapipe/selfie_segmentation][S-npm] | [mediapipe.dev/demo/selfie_segmentation][S-demo]
 
 Click on a solution link above for more information, including API and code
 snippets.
 
+### Supported plaforms:
+
+| Browser | Platform                | Notes                                  |
+| ------- | ----------------------- | -------------------------------------- |
+| Chrome  | Android / Windows / Mac | Pixel 4 and older unsupported. Fuschia |
+|         |                         | unsupported.                           |
+| Chrome  | iOS                     | Camera unavailable in Chrome on iOS.   |
+| Safari  | iPad/iPhone/Mac         | iOS and Safari on iPad / iPhone /      |
+|         |                         | MacBook                                |
+
 The quickest way to get acclimated is to look at the examples above. Each demo
 has a link to a [CodePen][codepen] so that you can edit the code and try it
 yourself. We have included a number of utility packages to help you get started:
@@ -67,33 +78,24 @@ affecting your work, restrict your request to a `<minor>` number. e.g.,
 [F-pg]: ../solutions/face_mesh#javascript-solution-api
 [Fd-pg]: ../solutions/face_detection#javascript-solution-api
 [H-pg]: ../solutions/hands#javascript-solution-api
+[Ob-pg]: ../solutions/objectron#javascript-solution-api
 [P-pg]: ../solutions/pose#javascript-solution-api
 [S-pg]: ../solutions/selfie_segmentation#javascript-solution-api
 [Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic
 [F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh
 [Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection
 [H-npm]: https://www.npmjs.com/package/@mediapipe/hands
+[Ob-npm]: https://www.npmjs.com/package/@mediapipe/objectron
 [P-npm]: https://www.npmjs.com/package/@mediapipe/pose
 [S-npm]: https://www.npmjs.com/package/@mediapipe/selfie_segmentation
 [draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils
 [cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils
 [ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils
-[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic
-[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh
-[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection
-[H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands
-[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose
-[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/selfie_segmentation
-[Ho-pen]: https://code.mediapipe.dev/codepen/holistic
-[F-pen]: https://code.mediapipe.dev/codepen/face_mesh
-[Fd-pen]: https://code.mediapipe.dev/codepen/face_detection
-[H-pen]: https://code.mediapipe.dev/codepen/hands
-[P-pen]: https://code.mediapipe.dev/codepen/pose
-[S-pen]: https://code.mediapipe.dev/codepen/selfie_segmentation
 [Ho-demo]: https://mediapipe.dev/demo/holistic
 [F-demo]: https://mediapipe.dev/demo/face_mesh
 [Fd-demo]: https://mediapipe.dev/demo/face_detection
 [H-demo]: https://mediapipe.dev/demo/hands
+[Ob-demo]: https://mediapipe.dev/demo/objectron
 [P-demo]: https://mediapipe.dev/demo/pose
 [S-demo]: https://mediapipe.dev/demo/selfie_segmentation
 [npm]: https://www.npmjs.com/package/@mediapipe

docs/getting_started/python_framework.md

@@ -74,7 +74,7 @@ Mapping\[str, Packet\]               | std::map<std::string, Packet> | create_st
 np.ndarray<br>(cv.mat and PIL.Image) | mp::ImageFrame                | create_image_frame(<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;format=ImageFormat.SRGB,<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;data=mat) | get_image_frame(packet)
 np.ndarray                           | mp::Matrix                    | create_matrix(data)                                                                                                                                          | get_matrix(packet)
 Google Proto Message                 | Google Proto Message          | create_proto(proto)                                                                                                                                          | get_proto(packet)
-List\[Proto\]                        | std::vector\<Proto\>          | create_proto_vector(proto_list)                                                                                                                              | get_proto_list(packet)
+List\[Proto\]                        | std::vector\<Proto\>          | n/a                                                                                                                                                          | get_proto_list(packet)
 
 It's not uncommon that users create custom C++ classes and and send those into
 the graphs and calculators. To allow the custom classes to be used in Python

docs/index.md

@@ -45,7 +45,7 @@ Hair Segmentation
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             | ✅                                                             |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
@@ -79,6 +79,13 @@ run code search using
 
 ## Publications
 
+*   [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
+    in Google Developers Blog
+*   [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html)
+    in Google Developers Blog
+*   [SignAll SDK: Sign language interface using MediaPipe is now available for
+    developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
+    in Google Developers Blog
 *   [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
     in Google AI Blog
 *   [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)

docs/solutions/face_mesh.md

@@ -278,6 +278,7 @@ Supported configuration options:
 import cv2
 import mediapipe as mp
 mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
 mp_face_mesh = mp.solutions.face_mesh
 
 # For static images:
@@ -301,9 +302,17 @@ with mp_face_mesh.FaceMesh(
       mp_drawing.draw_landmarks(
           image=annotated_image,
           landmark_list=face_landmarks,
-          connections=mp_face_mesh.FACE_CONNECTIONS,
+          connections=mp_face_mesh.FACEMESH_TESSELATION,
-          landmark_drawing_spec=drawing_spec,
+          landmark_drawing_spec=None,
-          connection_drawing_spec=drawing_spec)
+          connection_drawing_spec=mp_drawing_styles
+          .get_default_face_mesh_tesselation_style())
+      mp_drawing.draw_landmarks(
+          image=annotated_image,
+          landmark_list=face_landmarks,
+          connections=mp_face_mesh.FACEMESH_CONTOURS,
+          landmark_drawing_spec=None,
+          connection_drawing_spec=mp_drawing_styles
+          .get_default_face_mesh_contours_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
@@ -335,9 +344,17 @@ with mp_face_mesh.FaceMesh(
         mp_drawing.draw_landmarks(
             image=image,
             landmark_list=face_landmarks,
-            connections=mp_face_mesh.FACE_CONNECTIONS,
+            connections=mp_face_mesh.FACEMESH_TESSELATION,
-            landmark_drawing_spec=drawing_spec,
+            landmark_drawing_spec=None,
-            connection_drawing_spec=drawing_spec)
+            connection_drawing_spec=mp_drawing_styles
+            .get_default_face_mesh_tesselation_style())
+        mp_drawing.draw_landmarks(
+            image=image,
+            landmark_list=face_landmarks,
+            connections=mp_face_mesh.FACEMESH_CONTOURS,
+            landmark_drawing_spec=None,
+            connection_drawing_spec=mp_drawing_styles
+            .get_default_face_mesh_contours_style())
     cv2.imshow('MediaPipe FaceMesh', image)
     if cv2.waitKey(5) & 0xFF == 27:
       break
@@ -423,6 +440,200 @@ camera.start();
 </script>
 ```
 
+### Android Solution API
+
+Please first follow general
+[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
+to add MediaPipe Gradle dependencies, then try the FaceMash solution API in the
+companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
+following
+[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+and learn more in the usage example below.
+
+Supported configuration options:
+
+*   [staticImageMode](#static_image_mode)
+*   [maxNumFaces](#max_num_faces)
+*   runOnGpu: Run the pipeline and the model inference on GPU or CPU.
+
+#### Camera Input
+
+```java
+// For camera input and result rendering with OpenGL.
+FaceMeshOptions faceMeshOptions =
+    FaceMeshOptions.builder()
+        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
+        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setRunOnGpu(true).build();
+FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+facemesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+
+// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
+CameraInput cameraInput = new CameraInput(this);
+cameraInput.setNewFrameListener(
+    textureFrame -> facemesh.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+facemesh.setResultListener(
+    faceMeshResult -> {
+      NormalizedLandmark noseLandmark =
+          result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseLandmark.getX(), noseLandmark.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceMeshResult);
+      glSurfaceView.requestRender();
+    });
+
+// The runnable to start camera after the GLSurfaceView is attached.
+glSurfaceView.post(
+    () ->
+        cameraInput.start(
+            this,
+            facemesh.getGlContext(),
+            CameraInput.CameraFacing.FRONT,
+            glSurfaceView.getWidth(),
+            glSurfaceView.getHeight()));
+```
+
+#### Image Input
+
+```java
+// For reading images from gallery and drawing the output in an ImageView.
+FaceMeshOptions faceMeshOptions =
+    FaceMeshOptions.builder()
+        .setMode(FaceMeshOptions.STATIC_IMAGE_MODE)  // API soon to become
+        .setMaxNumFaces(1)                           // setStaticImageMode(true)
+        .setRunOnGpu(true).build();
+FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+
+// Connects MediaPipe FaceMesh to the user-defined ImageView instance that allows
+// users to have the custom drawing of the output landmarks on it.
+// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
+// as an example.
+FaceMeshResultImageView imageView = new FaceMeshResultImageView(this);
+facemesh.setResultListener(
+    faceMeshResult -> {
+      int width = faceMeshResult.inputBitmap().getWidth();
+      int height = faceMeshResult.inputBitmap().getHeight();
+      NormalizedLandmark noseLandmark =
+          result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
+              noseLandmark.getX() * width, noseLandmark.getY() * height));
+      // Request canvas drawing.
+      imageView.setFaceMeshResult(faceMeshResult);
+      runOnUiThread(() -> imageView.update());
+    });
+facemesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+
+// ActivityResultLauncher to get an image from the gallery as Bitmap.
+ActivityResultLauncher<Intent> imageGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null && result.getResultCode() == RESULT_OK) {
+            Bitmap bitmap = null;
+            try {
+              bitmap =
+                  MediaStore.Images.Media.getBitmap(
+                      this.getContentResolver(), resultIntent.getData());
+            } catch (IOException e) {
+              Log.e(TAG, "Bitmap reading error:" + e);
+            }
+            if (bitmap != null) {
+              facemesh.send(bitmap);
+            }
+          }
+        });
+Intent gallery = new Intent(
+    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
+imageGetter.launch(gallery);
+```
+
+#### Video Input
+
+```java
+// For video input and result rendering with OpenGL.
+FaceMeshOptions faceMeshOptions =
+    FaceMeshOptions.builder()
+        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
+        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setRunOnGpu(true).build();
+FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+facemesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+
+// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
+VideoInput videoInput = new VideoInput(this);
+videoInput.setNewFrameListener(
+    textureFrame -> facemesh.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+facemesh.setResultListener(
+    faceMeshResult -> {
+      NormalizedLandmark noseLandmark =
+          result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseLandmark.getX(), noseLandmark.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceMeshResult);
+      glSurfaceView.requestRender();
+    });
+
+ActivityResultLauncher<Intent> videoGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null) {
+            if (result.getResultCode() == RESULT_OK) {
+              glSurfaceView.post(
+                  () ->
+                      videoInput.start(
+                          this,
+                          resultIntent.getData(),
+                          facemesh.getGlContext(),
+                          glSurfaceView.getWidth(),
+                          glSurfaceView.getHeight()));
+            }
+          }
+        });
+Intent gallery =
+    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
+videoGetter.launch(gallery);
+```
+
 ## Example Apps
 
 Please first see general instructions for

docs/solutions/hands.md

@@ -219,6 +219,7 @@ Supported configuration options:
 import cv2
 import mediapipe as mp
 mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
 mp_hands = mp.solutions.hands
 
 # For static images:
@@ -248,7 +249,11 @@ with mp_hands.Hands(
           f'{hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP].y * image_height})'
       )
       mp_drawing.draw_landmarks(
-          annotated_image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+          annotated_image,
+          hand_landmarks,
+          mp_hands.HAND_CONNECTIONS,
+          mp_drawing_styles.get_default_hand_landmarks_style(),
+          mp_drawing_styles.get_default_hand_connections_style())
     cv2.imwrite(
         '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
 
@@ -278,7 +283,11 @@ with mp_hands.Hands(
     if results.multi_hand_landmarks:
       for hand_landmarks in results.multi_hand_landmarks:
         mp_drawing.draw_landmarks(
-            image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+            image,
+            hand_landmarks,
+            mp_hands.HAND_CONNECTIONS,
+            mp_drawing_styles.get_default_hand_landmarks_style(),
+            mp_drawing_styles.get_default_hand_connections_style())
     cv2.imshow('MediaPipe Hands', image)
     if cv2.waitKey(5) & 0xFF == 27:
       break
@@ -359,6 +368,200 @@ camera.start();
 </script>
 ```
 
+### Android Solution API
+
+Please first follow general
+[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
+to add MediaPipe Gradle dependencies, then try the Hands solution API in the
+companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
+following
+[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+and learn more in usage example below.
+
+Supported configuration options:
+
+*   [staticImageMode](#static_image_mode)
+*   [maxNumHands](#max_num_hands)
+*   runOnGpu: Run the pipeline and the model inference on GPU or CPU.
+
+#### Camera Input
+
+```java
+// For camera input and result rendering with OpenGL.
+HandsOptions handsOptions =
+    HandsOptions.builder()
+        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
+        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setRunOnGpu(true).build();
+Hands hands = new Hands(this, handsOptions);
+hands.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
+
+// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
+CameraInput cameraInput = new CameraInput(this);
+cameraInput.setNewFrameListener(
+    textureFrame -> hands.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<HandsResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<HandsResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, hands.getGlContext(), hands.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new HandsResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+hands.setResultListener(
+    handsResult -> {
+      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
+          handsResult, 0, HandLandmark.WRIST);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              wristLandmark.getX(), wristLandmark.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(handsResult);
+      glSurfaceView.requestRender();
+    });
+
+// The runnable to start camera after the GLSurfaceView is attached.
+glSurfaceView.post(
+    () ->
+        cameraInput.start(
+            this,
+            hands.getGlContext(),
+            CameraInput.CameraFacing.FRONT,
+            glSurfaceView.getWidth(),
+            glSurfaceView.getHeight()));
+```
+
+#### Image Input
+
+```java
+// For reading images from gallery and drawing the output in an ImageView.
+HandsOptions handsOptions =
+    HandsOptions.builder()
+        .setMode(HandsOptions.STATIC_IMAGE_MODE)  // API soon to become
+        .setMaxNumHands(1)                        // setStaticImageMode(true)
+        .setRunOnGpu(true).build();
+Hands hands = new Hands(this, handsOptions);
+
+// Connects MediaPipe Hands to the user-defined ImageView instance that allows
+// users to have the custom drawing of the output landmarks on it.
+// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
+// as an example.
+HandsResultImageView imageView = new HandsResultImageView(this);
+hands.setResultListener(
+    handsResult -> {
+      int width = handsResult.inputBitmap().getWidth();
+      int height = handsResult.inputBitmap().getHeight();
+      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
+          handsResult, 0, HandLandmark.WRIST);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Hand wrist coordinates (pixel values): x=%f, y=%f",
+              wristLandmark.getX() * width, wristLandmark.getY() * height));
+      // Request canvas drawing.
+      imageView.setHandsResult(handsResult);
+      runOnUiThread(() -> imageView.update());
+    });
+hands.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
+
+// ActivityResultLauncher to get an image from the gallery as Bitmap.
+ActivityResultLauncher<Intent> imageGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null && result.getResultCode() == RESULT_OK) {
+            Bitmap bitmap = null;
+            try {
+              bitmap =
+                  MediaStore.Images.Media.getBitmap(
+                      this.getContentResolver(), resultIntent.getData());
+            } catch (IOException e) {
+              Log.e(TAG, "Bitmap reading error:" + e);
+            }
+            if (bitmap != null) {
+              hands.send(bitmap);
+            }
+          }
+        });
+Intent gallery = new Intent(
+    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
+imageGetter.launch(gallery);
+```
+
+#### Video Input
+
+```java
+// For video input and result rendering with OpenGL.
+HandsOptions handsOptions =
+    HandsOptions.builder()
+        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
+        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setRunOnGpu(true).build();
+Hands hands = new Hands(this, handsOptions);
+hands.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
+
+// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
+VideoInput videoInput = new VideoInput(this);
+videoInput.setNewFrameListener(
+    textureFrame -> hands.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<HandsResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<HandsResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, hands.getGlContext(), hands.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new HandsResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+hands.setResultListener(
+    handsResult -> {
+      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
+          handsResult, 0, HandLandmark.WRIST);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              wristLandmark.getX(), wristLandmark.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(handsResult);
+      glSurfaceView.requestRender();
+    });
+
+ActivityResultLauncher<Intent> videoGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null) {
+            if (result.getResultCode() == RESULT_OK) {
+              glSurfaceView.post(
+                  () ->
+                      videoInput.start(
+                          this,
+                          resultIntent.getData(),
+                          hands.getGlContext(),
+                          glSurfaceView.getWidth(),
+                          glSurfaceView.getHeight()));
+            }
+          }
+        });
+Intent gallery =
+    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
+videoGetter.launch(gallery);
+```
+
 ## Example Apps
 
 Please first see general instructions for

docs/solutions/holistic.md

@@ -225,6 +225,7 @@ Supported configuration options:
 import cv2
 import mediapipe as mp
 mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
 mp_holistic = mp.solutions.holistic
 
 # For static images:
@@ -247,13 +248,18 @@ with mp_holistic.Holistic(
     # Draw pose, left and right hands, and face landmarks on the image.
     annotated_image = image.copy()
     mp_drawing.draw_landmarks(
-        annotated_image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
+        annotated_image,
+        results.face_landmarks,
+        mp_holistic.FACEMESH_TESSELATION,
+        landmark_drawing_spec=None,
+        connection_drawing_spec=mp_drawing_styles
+        .get_default_face_mesh_tesselation_style())
     mp_drawing.draw_landmarks(
-        annotated_image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+        annotated_image,
-    mp_drawing.draw_landmarks(
+        results.pose_landmarks,
-        annotated_image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+        mp_holistic.POSE_CONNECTIONS,
-    mp_drawing.draw_landmarks(
+        landmark_drawing_spec=mp_drawing_styles.
-        annotated_image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
+        get_default_pose_landmarks_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
     # Plot pose world landmarks.
     mp_drawing.plot_landmarks(
@@ -283,13 +289,18 @@ with mp_holistic.Holistic(
     image.flags.writeable = True
     image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
     mp_drawing.draw_landmarks(
-        image, results.face_landmarks, mp_holistic.FACE_CONNECTIONS)
+        image,
+        results.face_landmarks,
+        mp_holistic.FACEMESH_CONTOURS,
+        landmark_drawing_spec=None,
+        connection_drawing_spec=mp_drawing_styles
+        .get_default_face_mesh_contours_style())
     mp_drawing.draw_landmarks(
-        image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+        image,
-    mp_drawing.draw_landmarks(
+        results.pose_landmarks,
-        image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS)
+        mp_holistic.POSE_CONNECTIONS,
-    mp_drawing.draw_landmarks(
+        landmark_drawing_spec=mp_drawing_styles
-        image, results.pose_landmarks, mp_holistic.POSE_CONNECTIONS)
+        .get_default_pose_landmarks_style())
     cv2.imshow('MediaPipe Holistic', image)
     if cv2.waitKey(5) & 0xFF == 27:
       break

docs/solutions/objectron.md

@@ -224,29 +224,33 @@ where object detection simply runs on every image. Default to `0.99`.
 
 #### model_name
 
-Name of the model to use for predicting 3D bounding box landmarks. Currently supports
+Name of the model to use for predicting 3D bounding box landmarks. Currently
-`{'Shoe', 'Chair', 'Cup', 'Camera'}`.
+supports `{'Shoe', 'Chair', 'Cup', 'Camera'}`. Default to `Shoe`.
 
 #### focal_length
 
-Camera focal length `(fx, fy)`, by default is defined in
+By default, camera focal length defined in [NDC space](#ndc-space), i.e., `(fx,
-[NDC space](#ndc-space). To use focal length `(fx_pixel, fy_pixel)` in
+fy)`. Default to `(1.0, 1.0)`. To specify focal length in
-[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
+[pixel space](#pixel-space) instead, i.e., `(fx_pixel, fy_pixel)`, users should
-image_height)` to enable conversions inside the API. For further details about
+provide [`image_size`](#image_size) = `(image_width, image_height)` to enable
-NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
+conversions inside the API. For further details about NDC and pixel space,
+please see [Coordinate Systems](#coordinate-systems).
 
 #### principal_point
 
-Camera principal point `(px, py)`, by default is defined in
+By default, camera principal point defined in [NDC space](#ndc-space), i.e.,
-[NDC space](#ndc-space). To use principal point `(px_pixel, py_pixel)` in
+`(px, py)`. Default to `(0.0, 0.0)`. To specify principal point in
-[pixel space](#pixel-space), users should provide `image_size` = `(image_width,
+[pixel space](#pixel-space), i.e.,`(px_pixel, py_pixel)`, users should provide
-image_height)` to enable conversions inside the API. For further details about
+[`image_size`](#image_size) = `(image_width, image_height)` to enable
-NDC and pixel space, please see [Coordinate Systems](#coordinate-systems).
+conversions inside the API. For further details about NDC and pixel space,
+please see [Coordinate Systems](#coordinate-systems).
 
 #### image_size
 
-(**Optional**) size `(image_width, image_height)` of the input image, **ONLY**
+**Specify only when [`focal_length`](#focal_length) and
-needed when use `focal_length` and `principal_point` in pixel space.
+[`principal_point`](#principal_point) are specified in pixel space.**
+
+Size of the input image, i.e., `(image_width, image_height)`.
 
 ### Output
 
@@ -356,6 +360,89 @@ with mp_objectron.Objectron(static_image_mode=False,
 cap.release()
 ```
 
+## JavaScript Solution API
+
+Please first see general [introduction](../getting_started/javascript.md) on
+MediaPipe in JavaScript, then learn more in the companion [web demo](#resources)
+and the following usage example.
+
+Supported configuration options:
+
+*   [staticImageMode](#static_image_mode)
+*   [maxNumObjects](#max_num_objects)
+*   [minDetectionConfidence](#min_detection_confidence)
+*   [minTrackingConfidence](#min_tracking_confidence)
+*   [modelName](#model_name)
+*   [focalLength](#focal_length)
+*   [principalPoint](#principal_point)
+*   [imageSize](#image_size)
+
+```html
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/control_utils_3d.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/objectron/objectron.js" crossorigin="anonymous"></script>
+</head>
+
+<body>
+  <div class="container">
+    <video class="input_video"></video>
+    <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+  </div>
+</body>
+</html>
+```
+
+```javascript
+<script type="module">
+const videoElement = document.getElementsByClassName('input_video')[0];
+const canvasElement = document.getElementsByClassName('output_canvas')[0];
+const canvasCtx = canvasElement.getContext('2d');
+
+function onResults(results) {
+  canvasCtx.save();
+  canvasCtx.drawImage(
+      results.image, 0, 0, canvasElement.width, canvasElement.height);
+  if (!!results.objectDetections) {
+    for (const detectedObject of results.objectDetections) {
+      // Reformat keypoint information as landmarks, for easy drawing.
+      const landmarks: mpObjectron.Point2D[] =
+          detectedObject.keypoints.map(x => x.point2d);
+      // Draw bounding box.
+      drawingUtils.drawConnectors(canvasCtx, landmarks,
+          mpObjectron.BOX_CONNECTIONS, {color: '#FF0000'});
+      // Draw centroid.
+      drawingUtils.drawLandmarks(canvasCtx, [landmarks[0]], {color: '#FFFFFF'});
+    }
+  }
+  canvasCtx.restore();
+}
+
+const objectron = new Objectron({locateFile: (file) => {
+  return `https://cdn.jsdelivr.net/npm/@mediapipe/objectron/${file}`;
+}});
+objectron.setOptions({
+  modelName: 'Chair',
+  maxNumObjects: 3,
+});
+objectron.onResults(onResults);
+
+const camera = new Camera(videoElement, {
+  onFrame: async () => {
+    await objectron.send({image: videoElement});
+  },
+  width: 1280,
+  height: 720
+});
+camera.start();
+</script>
+```
+
 ## Example Apps
 
 Please first see general instructions for
@@ -442,7 +529,7 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > ```
 > and then run
 >
-> ```build
+> ```bash
 > bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
 > ```
 > INPUT_DIR should be the folder with initial asset .obj files to be processed,
@@ -561,11 +648,15 @@ py = -py_pixel * 2.0 / image_height + 1.0
     [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html)
 *   Google AI Blog:
     [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
-*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021
+*   Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the
+    Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in
+    CVPR 2021
 *   Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak
     Shape Supervision](https://arxiv.org/abs/2003.03522)
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
-    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020
+    ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth
+    Workshop on Computer Vision for AR/VR, CVPR 2020
 *   [Models and model cards](./models.md#objectron)
+*   [Web demo](https://code.mediapipe.dev/codepen/objectron)
 *   [Python Colab](https://mediapipe.page.link/objectron_py_colab)

docs/solutions/pose.md

@@ -30,7 +30,8 @@ overlay of digital content and information on top of the physical world in
 augmented reality.
 
 MediaPipe Pose is a ML solution for high-fidelity body pose tracking, inferring
-33 3D landmarks on the whole body from RGB video frames utilizing our
+33 3D landmarks and background segmentation mask on the whole body from RGB
+video frames utilizing our
 [BlazePose](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 research that also powers the
 [ML Kit Pose Detection API](https://developers.google.com/ml-kit/vision/pose-detection).
@@ -49,11 +50,11 @@ The solution utilizes a two-step detector-tracker ML pipeline, proven to be
 effective in our [MediaPipe Hands](./hands.md) and
 [MediaPipe Face Mesh](./face_mesh.md) solutions. Using a detector, the pipeline
 first locates the person/pose region-of-interest (ROI) within the frame. The
-tracker subsequently predicts the pose landmarks within the ROI using the
+tracker subsequently predicts the pose landmarks and segmentation mask within
-ROI-cropped frame as input. Note that for video use cases the detector is
+the ROI using the ROI-cropped frame as input. Note that for video use cases the
-invoked only as needed, i.e., for the very first frame and when the tracker
+detector is invoked only as needed, i.e., for the very first frame and when the
-could no longer identify body pose presence in the previous frame. For other
+tracker could no longer identify body pose presence in the previous frame. For
-frames the pipeline simply derives the ROI from the previous frame’s pose
+other frames the pipeline simply derives the ROI from the previous frame’s pose
 landmarks.
 
 The pipeline is implemented as a MediaPipe
@@ -129,16 +130,19 @@ hip midpoints.
 The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
 (see figure below).
 
-Please find more detail in the
-[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
-this [paper](https://arxiv.org/abs/2006.10204) and
-[the model card](./models.md#pose), and the attributes in each landmark
-[below](#pose_landmarks).
-
 ![pose_tracking_full_body_landmarks.png](../images/mobile/pose_tracking_full_body_landmarks.png) |
 :----------------------------------------------------------------------------------------------: |
 *Fig 4. 33 pose landmarks.*                                                                      |
 
+Optionally, MediaPipe Pose can predicts a full-body
+[segmentation mask](#segmentation_mask) represented as a two-class segmentation
+(human or background).
+
+Please find more detail in the
+[BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
+this [paper](https://arxiv.org/abs/2006.10204),
+[the model card](./models.md#pose) and the [Output](#output) section below.
+
 ## Solution APIs
 
 ### Cross-platform Configuration Options
@@ -167,6 +171,18 @@ If set to `true`, the solution filters pose landmarks across different input
 images to reduce jitter, but ignored if [static_image_mode](#static_image_mode)
 is also set to `true`. Default to `true`.
 
+#### enable_segmentation
+
+If set to `true`, in addition to the pose landmarks the solution also generates
+the segmentation mask. Default to `false`.
+
+#### smooth_segmentation
+
+If set to `true`, the solution filters segmentation masks across different input
+images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
+is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
+`true`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
@@ -211,6 +227,19 @@ the following:
 *   `visibility`: Identical to that defined in the corresponding
     [pose_landmarks](#pose_landmarks).
 
+#### segmentation_mask
+
+The output segmentation mask, predicted only when
+[enable_segmentation](#enable_segmentation) is set to `true`. The mask has the
+same width and height as the input image, and contains values in `[0.0, 1.0]`
+where `1.0` and `0.0` indicate high certainty of a "human" and "background"
+pixel respectively. Please refer to the platform-specific usage examples below
+for usage details.
+
+*Fig 6. Example of MediaPipe Pose segmentation mask.* |
+:-----------------------------------------------------------: |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
+
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
@@ -222,6 +251,8 @@ Supported configuration options:
 *   [static_image_mode](#static_image_mode)
 *   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
+*   [enable_segmentation](#enable_segmentation)
+*   [smooth_segmentation](#smooth_segmentation)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -229,13 +260,16 @@ Supported configuration options:
 import cv2
 import mediapipe as mp
 mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
 mp_pose = mp.solutions.pose
 
 # For static images:
 IMAGE_FILES = []
+BG_COLOR = (192, 192, 192) # gray
 with mp_pose.Pose(
     static_image_mode=True,
     model_complexity=2,
+    enable_segmentation=True,
     min_detection_confidence=0.5) as pose:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
@@ -247,13 +281,24 @@ with mp_pose.Pose(
       continue
     print(
         f'Nose coordinates: ('
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].x * image_width}, '
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].y * image_height})'
     )
-    # Draw pose landmarks on the image.
+
     annotated_image = image.copy()
+    # Draw segmentation on the image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
+    bg_image = np.zeros(image.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    annotated_image = np.where(condition, annotated_image, bg_image)
+    # Draw pose landmarks on the image.
     mp_drawing.draw_landmarks(
-        annotated_image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
+        annotated_image,
+        results.pose_landmarks,
+        mp_pose.POSE_CONNECTIONS,
+        landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
     # Plot pose world landmarks.
     mp_drawing.plot_landmarks(
@@ -283,7 +328,10 @@ with mp_pose.Pose(
     image.flags.writeable = True
     image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
     mp_drawing.draw_landmarks(
-        image, results.pose_landmarks, mp_pose.POSE_CONNECTIONS)
+        image,
+        results.pose_landmarks,
+        mp_pose.POSE_CONNECTIONS,
+        landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
     cv2.imshow('MediaPipe Pose', image)
     if cv2.waitKey(5) & 0xFF == 27:
       break
@@ -300,6 +348,8 @@ Supported configuration options:
 
 *   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
+*   [enableSegmentation](#enable_segmentation)
+*   [smoothSegmentation](#smooth_segmentation)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -319,6 +369,7 @@ Supported configuration options:
   <div class="container">
     <video class="input_video"></video>
     <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+    <div class="landmark-grid-container"></div>
   </div>
 </body>
 </html>
@@ -340,8 +391,20 @@ function onResults(results) {
 
   canvasCtx.save();
   canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(results.segmentationMask, 0, 0,
+                      canvasElement.width, canvasElement.height);
+
+  // Only overwrite existing pixels.
+  canvasCtx.globalCompositeOperation = 'source-in';
+  canvasCtx.fillStyle = '#00FF00';
+  canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
+
+  // Only overwrite missing pixels.
+  canvasCtx.globalCompositeOperation = 'destination-atop';
   canvasCtx.drawImage(
       results.image, 0, 0, canvasElement.width, canvasElement.height);
+
+  canvasCtx.globalCompositeOperation = 'source-over';
   drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
                  {color: '#00FF00', lineWidth: 4});
   drawLandmarks(canvasCtx, results.poseLandmarks,
@@ -357,6 +420,8 @@ const pose = new Pose({locateFile: (file) => {
 pose.setOptions({
   modelComplexity: 1,
   smoothLandmarks: true,
+  enableSegmentation: true,
+  smoothSegmentation: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });

docs/solutions/selfie_segmentation.md

@@ -96,6 +96,7 @@ Supported configuration options:
 ```python
 import cv2
 import mediapipe as mp
+import numpy as np
 mp_drawing = mp.solutions.drawing_utils
 mp_selfie_segmentation = mp.solutions.selfie_segmentation
 
@@ -261,7 +262,7 @@ to visualize its associated subgraphs, please see
     [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1DoeyGzMmWUsjfVgZfGGecrn7GKzYcEAo/view?usp=sharing)
     [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu:selfiesegmentationgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD)
 *   iOS target:
-    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](http:/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
+    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
 
 ### Desktop
 

docs/solutions/solutions.md

@@ -13,6 +13,9 @@ has_toc: false
 {:toc}
 ---
 
+MediaPipe offers open source cross-platform, customizable ML solutions for live
+and streaming media.
+
 <!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
 <!-- Whenever this table is updated, paste a copy to ../external_index.md. -->
 
@@ -29,7 +32,7 @@ has_toc: false
 [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)               | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               | ✅
 [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking)                       | ✅                                                               | ✅                                                       | ✅                                                       |                                                               |                                                               |
 [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             |                                                               |
+[Objectron](https://google.github.io/mediapipe/solutions/objectron)                             | ✅                                                               |                                                         | ✅                                                       | ✅                                                             | ✅                                                             |
 [KNIFT](https://google.github.io/mediapipe/solutions/knift)                                     | ✅                                                               |                                                         |                                                         |                                                               |                                                               |
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |                                                                 |                                                         | ✅                                                       |                                                               |                                                               |

mediapipe/calculators/core/BUILD

@@ -140,6 +140,16 @@ mediapipe_proto_library(
     ],
 )
 
+mediapipe_proto_library(
+    name = "graph_profile_calculator_proto",
+    srcs = ["graph_profile_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 cc_library(
     name = "add_header_calculator",
     srcs = ["add_header_calculator.cc"],
@@ -1200,3 +1210,45 @@ cc_test(
         "@com_google_absl//absl/strings",
     ],
 )
+
+cc_library(
+    name = "graph_profile_calculator",
+    srcs = ["graph_profile_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":graph_profile_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_profile_cc_proto",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "graph_profile_calculator_test",
+    srcs = ["graph_profile_calculator_test.cc"],
+    deps = [
+        ":graph_profile_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_profile_cc_proto",
+        "//mediapipe/framework:test_calculators",
+        "//mediapipe/framework/deps:clock",
+        "//mediapipe/framework/deps:message_matchers",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:threadpool",
+        "//mediapipe/framework/tool:simulation_clock_executor",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/time",
+    ],
+)

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -24,6 +24,9 @@
 
 namespace mediapipe {
 
+constexpr char kDataTag[] = "DATA";
+constexpr char kHeaderTag[] = "HEADER";
+
 class AddHeaderCalculatorTest : public ::testing::Test {};
 
 TEST_F(AddHeaderCalculatorTest, HeaderStream) {
@@ -36,11 +39,11 @@ TEST_F(AddHeaderCalculatorTest, HeaderStream) {
   CalculatorRunner runner(node);
 
   // Set header and add 5 packets.
-  runner.MutableInputs()->Tag("HEADER").header =
+  runner.MutableInputs()->Tag(kHeaderTag).header =
       Adopt(new std::string("my_header"));
   for (int i = 0; i < 5; ++i) {
     Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
-    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+    runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
   }
 
   // Run calculator.
@@ -85,13 +88,14 @@ TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
   CalculatorRunner runner(node);
 
   // Set header and add 5 packets.
-  runner.MutableInputs()->Tag("HEADER").header =
+  runner.MutableInputs()->Tag(kHeaderTag).header =
       Adopt(new std::string("my_header"));
-  runner.MutableInputs()->Tag("HEADER").packets.push_back(
+  runner.MutableInputs()
-      Adopt(new std::string("not allowed")));
+      ->Tag(kHeaderTag)
+      .packets.push_back(Adopt(new std::string("not allowed")));
   for (int i = 0; i < 5; ++i) {
     Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
-    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+    runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
   }
 
   // Run calculator.
@@ -108,11 +112,11 @@ TEST_F(AddHeaderCalculatorTest, InputSidePacket) {
   CalculatorRunner runner(node);
 
   // Set header and add 5 packets.
-  runner.MutableSidePackets()->Tag("HEADER") =
+  runner.MutableSidePackets()->Tag(kHeaderTag) =
       Adopt(new std::string("my_header"));
   for (int i = 0; i < 5; ++i) {
     Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
-    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+    runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
   }
 
   // Run calculator.
@@ -143,13 +147,13 @@ TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) {
   CalculatorRunner runner(node);
 
   // Set both headers and add 5 packets.
-  runner.MutableSidePackets()->Tag("HEADER") =
+  runner.MutableSidePackets()->Tag(kHeaderTag) =
       Adopt(new std::string("my_header"));
-  runner.MutableSidePackets()->Tag("HEADER") =
+  runner.MutableSidePackets()->Tag(kHeaderTag) =
       Adopt(new std::string("my_header"));
   for (int i = 0; i < 5; ++i) {
     Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
-    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+    runner.MutableInputs()->Tag(kDataTag).packets.push_back(packet);
   }
 
   // Run should fail because header can only be provided one way.

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -42,4 +42,9 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
 REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
 
+// A calculator to process std::vector<std::vector<Matrix>>.
+typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
+    BeginLoopMatrixVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/counting_source_calculator.cc

@@ -19,6 +19,13 @@
 
 namespace mediapipe {
 
+constexpr char kIncrementTag[] = "INCREMENT";
+constexpr char kInitialValueTag[] = "INITIAL_VALUE";
+constexpr char kBatchSizeTag[] = "BATCH_SIZE";
+constexpr char kErrorCountTag[] = "ERROR_COUNT";
+constexpr char kMaxCountTag[] = "MAX_COUNT";
+constexpr char kErrorOnOpenTag[] = "ERROR_ON_OPEN";
+
 // Source calculator that produces MAX_COUNT*BATCH_SIZE int packets of
 // sequential numbers from INITIAL_VALUE (default 0) with a common
 // difference of INCREMENT (default 1) between successive numbers (with
@@ -33,53 +40,53 @@ class CountingSourceCalculator : public CalculatorBase {
   static absl::Status GetContract(CalculatorContract* cc) {
     cc->Outputs().Index(0).Set<int>();
 
-    if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN")) {
+    if (cc->InputSidePackets().HasTag(kErrorOnOpenTag)) {
-      cc->InputSidePackets().Tag("ERROR_ON_OPEN").Set<bool>();
+      cc->InputSidePackets().Tag(kErrorOnOpenTag).Set<bool>();
     }
 
-    RET_CHECK(cc->InputSidePackets().HasTag("MAX_COUNT") ||
+    RET_CHECK(cc->InputSidePackets().HasTag(kMaxCountTag) ||
-              cc->InputSidePackets().HasTag("ERROR_COUNT"));
+              cc->InputSidePackets().HasTag(kErrorCountTag));
-    if (cc->InputSidePackets().HasTag("MAX_COUNT")) {
+    if (cc->InputSidePackets().HasTag(kMaxCountTag)) {
-      cc->InputSidePackets().Tag("MAX_COUNT").Set<int>();
+      cc->InputSidePackets().Tag(kMaxCountTag).Set<int>();
     }
-    if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
+    if (cc->InputSidePackets().HasTag(kErrorCountTag)) {
-      cc->InputSidePackets().Tag("ERROR_COUNT").Set<int>();
+      cc->InputSidePackets().Tag(kErrorCountTag).Set<int>();
     }
 
-    if (cc->InputSidePackets().HasTag("BATCH_SIZE")) {
+    if (cc->InputSidePackets().HasTag(kBatchSizeTag)) {
-      cc->InputSidePackets().Tag("BATCH_SIZE").Set<int>();
+      cc->InputSidePackets().Tag(kBatchSizeTag).Set<int>();
     }
-    if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) {
+    if (cc->InputSidePackets().HasTag(kInitialValueTag)) {
-      cc->InputSidePackets().Tag("INITIAL_VALUE").Set<int>();
+      cc->InputSidePackets().Tag(kInitialValueTag).Set<int>();
     }
-    if (cc->InputSidePackets().HasTag("INCREMENT")) {
+    if (cc->InputSidePackets().HasTag(kIncrementTag)) {
-      cc->InputSidePackets().Tag("INCREMENT").Set<int>();
+      cc->InputSidePackets().Tag(kIncrementTag).Set<int>();
     }
     return absl::OkStatus();
   }
 
   absl::Status Open(CalculatorContext* cc) override {
-    if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN") &&
+    if (cc->InputSidePackets().HasTag(kErrorOnOpenTag) &&
-        cc->InputSidePackets().Tag("ERROR_ON_OPEN").Get<bool>()) {
+        cc->InputSidePackets().Tag(kErrorOnOpenTag).Get<bool>()) {
       return absl::NotFoundError("expected error");
     }
-    if (cc->InputSidePackets().HasTag("ERROR_COUNT")) {
+    if (cc->InputSidePackets().HasTag(kErrorCountTag)) {
-      error_count_ = cc->InputSidePackets().Tag("ERROR_COUNT").Get<int>();
+      error_count_ = cc->InputSidePackets().Tag(kErrorCountTag).Get<int>();
       RET_CHECK_LE(0, error_count_);
     }
-    if (cc->InputSidePackets().HasTag("MAX_COUNT")) {
+    if (cc->InputSidePackets().HasTag(kMaxCountTag)) {
-      max_count_ = cc->InputSidePackets().Tag("MAX_COUNT").Get<int>();
+      max_count_ = cc->InputSidePackets().Tag(kMaxCountTag).Get<int>();
       RET_CHECK_LE(0, max_count_);
     }
-    if (cc->InputSidePackets().HasTag("BATCH_SIZE")) {
+    if (cc->InputSidePackets().HasTag(kBatchSizeTag)) {
-      batch_size_ = cc->InputSidePackets().Tag("BATCH_SIZE").Get<int>();
+      batch_size_ = cc->InputSidePackets().Tag(kBatchSizeTag).Get<int>();
       RET_CHECK_LT(0, batch_size_);
     }
-    if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) {
+    if (cc->InputSidePackets().HasTag(kInitialValueTag)) {
-      counter_ = cc->InputSidePackets().Tag("INITIAL_VALUE").Get<int>();
+      counter_ = cc->InputSidePackets().Tag(kInitialValueTag).Get<int>();
     }
-    if (cc->InputSidePackets().HasTag("INCREMENT")) {
+    if (cc->InputSidePackets().HasTag(kIncrementTag)) {
-      increment_ = cc->InputSidePackets().Tag("INCREMENT").Get<int>();
+      increment_ = cc->InputSidePackets().Tag(kIncrementTag).Get<int>();
       RET_CHECK_LT(0, increment_);
     }
     RET_CHECK(error_count_ >= 0 || max_count_ >= 0);

mediapipe/calculators/core/dequantize_byte_array_calculator.cc

@@ -35,11 +35,14 @@
 //   }
 namespace mediapipe {
 
+constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
+constexpr char kEncodedTag[] = "ENCODED";
+
 class DequantizeByteArrayCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("ENCODED").Set<std::string>();
+    cc->Inputs().Tag(kEncodedTag).Set<std::string>();
-    cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
+    cc->Outputs().Tag(kFloatVectorTag).Set<std::vector<float>>();
     return absl::OkStatus();
   }
 
@@ -66,7 +69,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
 
   absl::Status Process(CalculatorContext* cc) final {
     const std::string& encoded =
-        cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
+        cc->Inputs().Tag(kEncodedTag).Value().Get<std::string>();
     std::vector<float> float_vector;
     float_vector.reserve(encoded.length());
     for (int i = 0; i < encoded.length(); ++i) {
@@ -74,7 +77,7 @@ class DequantizeByteArrayCalculator : public CalculatorBase {
           static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
     }
     cc->Outputs()
-        .Tag("FLOAT_VECTOR")
+        .Tag(kFloatVectorTag)
         .AddPacket(MakePacket<std::vector<float>>(float_vector)
                        .At(cc->InputTimestamp()));
     return absl::OkStatus();

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -25,6 +25,9 @@
 
 namespace mediapipe {
 
+constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
+constexpr char kEncodedTag[] = "ENCODED";
+
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
       ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
@@ -39,8 +42,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
       )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
-  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+  runner.MutableInputs()
-      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+      ->Tag(kEncodedTag)
+      .packets.push_back(
+          MakePacket<std::string>(empty_string).At(Timestamp(0)));
   auto status = runner.Run();
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(
@@ -64,8 +69,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
       )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
-  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+  runner.MutableInputs()
-      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+      ->Tag(kEncodedTag)
+      .packets.push_back(
+          MakePacket<std::string>(empty_string).At(Timestamp(0)));
   auto status = runner.Run();
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(
@@ -89,8 +96,10 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
       )pb");
   CalculatorRunner runner(node_config);
   std::string empty_string;
-  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+  runner.MutableInputs()
-      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+      ->Tag(kEncodedTag)
+      .packets.push_back(
+          MakePacket<std::string>(empty_string).At(Timestamp(0)));
   auto status = runner.Run();
   EXPECT_FALSE(status.ok());
   EXPECT_THAT(
@@ -114,14 +123,16 @@ TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
       )pb");
   CalculatorRunner runner(node_config);
   unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
-  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+  runner.MutableInputs()
-      MakePacket<std::string>(
+      ->Tag(kEncodedTag)
-          std::string(reinterpret_cast<char const*>(input), 4))
+      .packets.push_back(
-          .At(Timestamp(0)));
+          MakePacket<std::string>(
+              std::string(reinterpret_cast<char const*>(input), 4))
+              .At(Timestamp(0)));
   auto status = runner.Run();
   MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& outputs =
-      runner.Outputs().Tag("FLOAT_VECTOR").packets;
+      runner.Outputs().Tag(kFloatVectorTag).packets;
   EXPECT_EQ(1, outputs.size());
   const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
   ASSERT_FALSE(result.empty());

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -24,6 +24,11 @@
 
 namespace mediapipe {
 
+constexpr char kFinishedTag[] = "FINISHED";
+constexpr char kAllowTag[] = "ALLOW";
+constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
+constexpr char kOptionsTag[] = "OPTIONS";
+
 // FlowLimiterCalculator is used to limit the number of frames in flight
 // by dropping input frames when necessary.
 //
@@ -69,16 +74,19 @@ class FlowLimiterCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
     auto& side_inputs = cc->InputSidePackets();
-    side_inputs.Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
+    side_inputs.Tag(kOptionsTag).Set<FlowLimiterCalculatorOptions>().Optional();
-    cc->Inputs().Tag("OPTIONS").Set<FlowLimiterCalculatorOptions>().Optional();
+    cc->Inputs()
+        .Tag(kOptionsTag)
+        .Set<FlowLimiterCalculatorOptions>()
+        .Optional();
     RET_CHECK_GE(cc->Inputs().NumEntries(""), 1);
     for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) {
       cc->Inputs().Get("", i).SetAny();
       cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
     }
     cc->Inputs().Get("FINISHED", 0).SetAny();
-    cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>().Optional();
+    cc->InputSidePackets().Tag(kMaxInFlightTag).Set<int>().Optional();
-    cc->Outputs().Tag("ALLOW").Set<bool>().Optional();
+    cc->Outputs().Tag(kAllowTag).Set<bool>().Optional();
     cc->SetInputStreamHandler("ImmediateInputStreamHandler");
     cc->SetProcessTimestampBounds(true);
     return absl::OkStatus();
@@ -87,9 +95,9 @@ class FlowLimiterCalculator : public CalculatorBase {
   absl::Status Open(CalculatorContext* cc) final {
     options_ = cc->Options<FlowLimiterCalculatorOptions>();
     options_ = tool::RetrieveOptions(options_, cc->InputSidePackets());
-    if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
+    if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
       options_.set_max_in_flight(
-          cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>());
+          cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>());
     }
     input_queues_.resize(cc->Inputs().NumEntries(""));
     RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
@@ -104,8 +112,8 @@ class FlowLimiterCalculator : public CalculatorBase {
 
   // Outputs a packet indicating whether a frame was sent or dropped.
   void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
-    if (cc->Outputs().HasTag("ALLOW")) {
+    if (cc->Outputs().HasTag(kAllowTag)) {
-      cc->Outputs().Tag("ALLOW").AddPacket(MakePacket<bool>(allow).At(ts));
+      cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
     }
   }
 
@@ -155,7 +163,7 @@ class FlowLimiterCalculator : public CalculatorBase {
     options_ = tool::RetrieveOptions(options_, cc->Inputs());
 
     // Process the FINISHED input stream.
-    Packet finished_packet = cc->Inputs().Tag("FINISHED").Value();
+    Packet finished_packet = cc->Inputs().Tag(kFinishedTag).Value();
     if (finished_packet.Timestamp() == cc->InputTimestamp()) {
       while (!frames_in_flight_.empty() &&
              frames_in_flight_.front() <= finished_packet.Timestamp()) {
@@ -210,8 +218,8 @@ class FlowLimiterCalculator : public CalculatorBase {
       Timestamp bound =
           cc->Inputs().Get("", 0).Value().Timestamp().NextAllowedInStream();
       SetNextTimestampBound(bound, &cc->Outputs().Get("", 0));
-      if (cc->Outputs().HasTag("ALLOW")) {
+      if (cc->Outputs().HasTag(kAllowTag)) {
-        SetNextTimestampBound(bound, &cc->Outputs().Tag("ALLOW"));
+        SetNextTimestampBound(bound, &cc->Outputs().Tag(kAllowTag));
       }
     }
 

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -36,6 +36,13 @@
 namespace mediapipe {
 
 namespace {
+
+constexpr char kDropTimestampsTag[] = "DROP_TIMESTAMPS";
+constexpr char kClockTag[] = "CLOCK";
+constexpr char kWarmupTimeTag[] = "WARMUP_TIME";
+constexpr char kSleepTimeTag[] = "SLEEP_TIME";
+constexpr char kPacketTag[] = "PACKET";
+
 // A simple Semaphore for synchronizing test threads.
 class AtomicSemaphore {
  public:
@@ -204,17 +211,17 @@ TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) {
 class SleepCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("PACKET").SetAny();
+    cc->Inputs().Tag(kPacketTag).SetAny();
-    cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
+    cc->Outputs().Tag(kPacketTag).SetSameAs(&cc->Inputs().Tag(kPacketTag));
-    cc->InputSidePackets().Tag("SLEEP_TIME").Set<int64>();
+    cc->InputSidePackets().Tag(kSleepTimeTag).Set<int64>();
-    cc->InputSidePackets().Tag("WARMUP_TIME").Set<int64>();
+    cc->InputSidePackets().Tag(kWarmupTimeTag).Set<int64>();
-    cc->InputSidePackets().Tag("CLOCK").Set<mediapipe::Clock*>();
+    cc->InputSidePackets().Tag(kClockTag).Set<mediapipe::Clock*>();
     cc->SetTimestampOffset(0);
     return absl::OkStatus();
   }
 
   absl::Status Open(CalculatorContext* cc) final {
-    clock_ = cc->InputSidePackets().Tag("CLOCK").Get<mediapipe::Clock*>();
+    clock_ = cc->InputSidePackets().Tag(kClockTag).Get<mediapipe::Clock*>();
     return absl::OkStatus();
   }
 
@@ -222,10 +229,12 @@ class SleepCalculator : public CalculatorBase {
     ++packet_count;
     absl::Duration sleep_time = absl::Microseconds(
         packet_count == 1
-            ? cc->InputSidePackets().Tag("WARMUP_TIME").Get<int64>()
+            ? cc->InputSidePackets().Tag(kWarmupTimeTag).Get<int64>()
-            : cc->InputSidePackets().Tag("SLEEP_TIME").Get<int64>());
+            : cc->InputSidePackets().Tag(kSleepTimeTag).Get<int64>());
     clock_->Sleep(sleep_time);
-    cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value());
+    cc->Outputs()
+        .Tag(kPacketTag)
+        .AddPacket(cc->Inputs().Tag(kPacketTag).Value());
     return absl::OkStatus();
   }
 
@@ -240,24 +249,27 @@ REGISTER_CALCULATOR(SleepCalculator);
 class DropCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("PACKET").SetAny();
+    cc->Inputs().Tag(kPacketTag).SetAny();
-    cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET"));
+    cc->Outputs().Tag(kPacketTag).SetSameAs(&cc->Inputs().Tag(kPacketTag));
-    cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Set<bool>();
+    cc->InputSidePackets().Tag(kDropTimestampsTag).Set<bool>();
     cc->SetProcessTimestampBounds(true);
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) final {
-    if (!cc->Inputs().Tag("PACKET").Value().IsEmpty()) {
+    if (!cc->Inputs().Tag(kPacketTag).Value().IsEmpty()) {
       ++packet_count;
     }
     bool drop = (packet_count == 3);
-    if (!drop && !cc->Inputs().Tag("PACKET").Value().IsEmpty()) {
+    if (!drop && !cc->Inputs().Tag(kPacketTag).Value().IsEmpty()) {
-      cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value());
+      cc->Outputs()
+          .Tag(kPacketTag)
+          .AddPacket(cc->Inputs().Tag(kPacketTag).Value());
     }
-    if (!drop || !cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Get<bool>()) {
+    if (!drop || !cc->InputSidePackets().Tag(kDropTimestampsTag).Get<bool>()) {
-      cc->Outputs().Tag("PACKET").SetNextTimestampBound(
+      cc->Outputs()
-          cc->InputTimestamp().NextAllowedInStream());
+          .Tag(kPacketTag)
+          .SetNextTimestampBound(cc->InputTimestamp().NextAllowedInStream());
     }
     return absl::OkStatus();
   }

mediapipe/calculators/core/gate_calculator.cc

@@ -21,6 +21,11 @@
 namespace mediapipe {
 
 namespace {
+
+constexpr char kStateChangeTag[] = "STATE_CHANGE";
+constexpr char kDisallowTag[] = "DISALLOW";
+constexpr char kAllowTag[] = "ALLOW";
+
 enum GateState {
   GATE_UNINITIALIZED,
   GATE_ALLOW,
@@ -59,8 +64,9 @@ std::string ToString(GateState state) {
 // ALLOW or DISALLOW can also be specified as an input side packet. The rules
 // for evaluation remain the same as above.
 //
-// ALLOW/DISALLOW inputs must be specified either using input stream or
+// ALLOW/DISALLOW inputs must be specified either using input stream or via
-// via input side packet but not both.
+// input side packet but not both. If neither is specified, the behavior is then
+// determined by the "allow" field in the calculator options.
 //
 // Intended to be used with the default input stream handler, which synchronizes
 // all data input streams with the ALLOW/DISALLOW control input stream.
@@ -83,30 +89,33 @@ class GateCalculator : public CalculatorBase {
   GateCalculator() {}
 
   static absl::Status CheckAndInitAllowDisallowInputs(CalculatorContract* cc) {
-    bool input_via_side_packet = cc->InputSidePackets().HasTag("ALLOW") ||
+    bool input_via_side_packet = cc->InputSidePackets().HasTag(kAllowTag) ||
-                                 cc->InputSidePackets().HasTag("DISALLOW");
+                                 cc->InputSidePackets().HasTag(kDisallowTag);
     bool input_via_stream =
-        cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW");
+        cc->Inputs().HasTag(kAllowTag) || cc->Inputs().HasTag(kDisallowTag);
-    // Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW
-    // input.
-    RET_CHECK(input_via_side_packet ^ input_via_stream);
 
+    // Only one of input_side_packet or input_stream may specify
+    // ALLOW/DISALLOW input.
     if (input_via_side_packet) {
-      RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^
+      RET_CHECK(!input_via_stream);
-                cc->InputSidePackets().HasTag("DISALLOW"));
+      RET_CHECK(cc->InputSidePackets().HasTag(kAllowTag) ^
+                cc->InputSidePackets().HasTag(kDisallowTag));
 
-      if (cc->InputSidePackets().HasTag("ALLOW")) {
+      if (cc->InputSidePackets().HasTag(kAllowTag)) {
-        cc->InputSidePackets().Tag("ALLOW").Set<bool>();
+        cc->InputSidePackets().Tag(kAllowTag).Set<bool>().Optional();
       } else {
-        cc->InputSidePackets().Tag("DISALLOW").Set<bool>();
+        cc->InputSidePackets().Tag(kDisallowTag).Set<bool>().Optional();
       }
-    } else {
+    }
-      RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
+    if (input_via_stream) {
+      RET_CHECK(!input_via_side_packet);
+      RET_CHECK(cc->Inputs().HasTag(kAllowTag) ^
+                cc->Inputs().HasTag(kDisallowTag));
 
-      if (cc->Inputs().HasTag("ALLOW")) {
+      if (cc->Inputs().HasTag(kAllowTag)) {
-        cc->Inputs().Tag("ALLOW").Set<bool>();
+        cc->Inputs().Tag(kAllowTag).Set<bool>();
       } else {
-        cc->Inputs().Tag("DISALLOW").Set<bool>();
+        cc->Inputs().Tag(kDisallowTag).Set<bool>();
       }
     }
     return absl::OkStatus();
@@ -125,23 +134,22 @@ class GateCalculator : public CalculatorBase {
       cc->Outputs().Get("", i).SetSameAs(&cc->Inputs().Get("", i));
     }
 
-    if (cc->Outputs().HasTag("STATE_CHANGE")) {
+    if (cc->Outputs().HasTag(kStateChangeTag)) {
-      cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
+      cc->Outputs().Tag(kStateChangeTag).Set<bool>();
     }
 
     return absl::OkStatus();
   }
 
   absl::Status Open(CalculatorContext* cc) final {
-    use_side_packet_for_allow_disallow_ = false;
+    if (cc->InputSidePackets().HasTag(kAllowTag)) {
-    if (cc->InputSidePackets().HasTag("ALLOW")) {
       use_side_packet_for_allow_disallow_ = true;
       allow_by_side_packet_decision_ =
-          cc->InputSidePackets().Tag("ALLOW").Get<bool>();
+          cc->InputSidePackets().Tag(kAllowTag).Get<bool>();
-    } else if (cc->InputSidePackets().HasTag("DISALLOW")) {
+    } else if (cc->InputSidePackets().HasTag(kDisallowTag)) {
       use_side_packet_for_allow_disallow_ = true;
       allow_by_side_packet_decision_ =
-          !cc->InputSidePackets().Tag("DISALLOW").Get<bool>();
+          !cc->InputSidePackets().Tag(kDisallowTag).Get<bool>();
     }
 
     cc->SetOffset(TimestampDiff(0));
@@ -152,26 +160,34 @@ class GateCalculator : public CalculatorBase {
     const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
     empty_packets_as_allow_ = options.empty_packets_as_allow();
 
+    if (!use_side_packet_for_allow_disallow_ &&
+        !cc->Inputs().HasTag(kAllowTag) && !cc->Inputs().HasTag(kDisallowTag)) {
+      use_option_for_allow_disallow_ = true;
+      allow_by_option_decision_ = options.allow();
+    }
+
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) final {
     bool allow = empty_packets_as_allow_;
-    if (use_side_packet_for_allow_disallow_) {
+    if (use_option_for_allow_disallow_) {
+      allow = allow_by_option_decision_;
+    } else if (use_side_packet_for_allow_disallow_) {
       allow = allow_by_side_packet_decision_;
     } else {
-      if (cc->Inputs().HasTag("ALLOW") &&
+      if (cc->Inputs().HasTag(kAllowTag) &&
-          !cc->Inputs().Tag("ALLOW").IsEmpty()) {
+          !cc->Inputs().Tag(kAllowTag).IsEmpty()) {
-        allow = cc->Inputs().Tag("ALLOW").Get<bool>();
+        allow = cc->Inputs().Tag(kAllowTag).Get<bool>();
       }
-      if (cc->Inputs().HasTag("DISALLOW") &&
+      if (cc->Inputs().HasTag(kDisallowTag) &&
-          !cc->Inputs().Tag("DISALLOW").IsEmpty()) {
+          !cc->Inputs().Tag(kDisallowTag).IsEmpty()) {
-        allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
+        allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
       }
     }
     const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
 
-    if (cc->Outputs().HasTag("STATE_CHANGE")) {
+    if (cc->Outputs().HasTag(kStateChangeTag)) {
       if (last_gate_state_ != GATE_UNINITIALIZED &&
           last_gate_state_ != new_gate_state) {
         VLOG(2) << "State transition in " << cc->NodeName() << " @ "
@@ -179,7 +195,7 @@ class GateCalculator : public CalculatorBase {
                 << ToString(last_gate_state_) << " to "
                 << ToString(new_gate_state);
         cc->Outputs()
-            .Tag("STATE_CHANGE")
+            .Tag(kStateChangeTag)
             .AddPacket(MakePacket<bool>(allow).At(cc->InputTimestamp()));
       }
     }
@@ -211,8 +227,10 @@ class GateCalculator : public CalculatorBase {
   GateState last_gate_state_ = GATE_UNINITIALIZED;
   int num_data_streams_;
   bool empty_packets_as_allow_;
-  bool use_side_packet_for_allow_disallow_;
+  bool use_side_packet_for_allow_disallow_ = false;
   bool allow_by_side_packet_decision_;
+  bool use_option_for_allow_disallow_ = false;
+  bool allow_by_option_decision_;
 };
 REGISTER_CALCULATOR(GateCalculator);
 

mediapipe/calculators/core/gate_calculator.proto

@@ -29,4 +29,8 @@ message GateCalculatorOptions {
   // disallowing the corresponding packets in the data input streams. Setting
   // this option to true inverts that, allowing the data packets to go through.
   optional bool empty_packets_as_allow = 1;
+
+  // Whether to allow or disallow the input streams to pass when no
+  // ALLOW/DISALLOW input or side input is specified.
+  optional bool allow = 2 [default = false];
 }

mediapipe/calculators/core/gate_calculator_test.cc

@@ -22,6 +22,9 @@ namespace mediapipe {
 
 namespace {
 
+constexpr char kDisallowTag[] = "DISALLOW";
+constexpr char kAllowTag[] = "ALLOW";
+
 class GateCalculatorTest : public ::testing::Test {
  protected:
   // Helper to run a graph and return status.
@@ -110,6 +113,68 @@ TEST_F(GateCalculatorTest, InvalidInputs) {
   )")));
 }
 
+TEST_F(GateCalculatorTest, AllowByALLOWOptionToTrue) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        output_stream: "test_output"
+        options: {
+          [mediapipe.GateCalculatorOptions.ext] {
+            allow: true
+          }
+        }
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, false);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(2, output.size());
+  EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value());
+  EXPECT_EQ(kTimestampValue1, output[1].Timestamp().Value());
+  EXPECT_EQ(true, output[0].Get<bool>());
+  EXPECT_EQ(false, output[1].Get<bool>());
+}
+
+TEST_F(GateCalculatorTest, DisallowByALLOWOptionSetToFalse) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        output_stream: "test_output"
+        options: {
+          [mediapipe.GateCalculatorOptions.ext] {
+            allow: false
+          }
+        }
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, false);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(0, output.size());
+}
+
+TEST_F(GateCalculatorTest, DisallowByALLOWOptionNotSet) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        output_stream: "test_output"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, false);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(0, output.size());
+}
+
 TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) {
   SetRunner(R"(
         calculator: "GateCalculator"
@@ -117,7 +182,7 @@ TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) {
         input_stream: "test_input"
         output_stream: "test_output"
   )");
-  runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(true));
+  runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(true));
 
   constexpr int64 kTimestampValue0 = 42;
   RunTimeStep(kTimestampValue0, true);
@@ -139,7 +204,7 @@ TEST_F(GateCalculatorTest, AllowByDisallowSidePacketSetToFalse) {
         input_stream: "test_input"
         output_stream: "test_output"
   )");
-  runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(false));
+  runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(false));
 
   constexpr int64 kTimestampValue0 = 42;
   RunTimeStep(kTimestampValue0, true);
@@ -161,7 +226,7 @@ TEST_F(GateCalculatorTest, DisallowByALLOWSidePacketSetToFalse) {
         input_stream: "test_input"
         output_stream: "test_output"
   )");
-  runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
+  runner()->MutableSidePackets()->Tag(kAllowTag) = Adopt(new bool(false));
 
   constexpr int64 kTimestampValue0 = 42;
   RunTimeStep(kTimestampValue0, true);
@@ -179,7 +244,7 @@ TEST_F(GateCalculatorTest, DisallowByDISALLOWSidePacketSetToTrue) {
         input_stream: "test_input"
         output_stream: "test_output"
   )");
-  runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(true));
+  runner()->MutableSidePackets()->Tag(kDisallowTag) = Adopt(new bool(true));
 
   constexpr int64 kTimestampValue0 = 42;
   RunTimeStep(kTimestampValue0, true);

mediapipe/calculators/core/graph_profile_calculator.cc

@@ -0,0 +1,70 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+
+#include "mediapipe/calculators/core/graph_profile_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// This calculator periodically copies the GraphProfile from
+// mediapipe::GraphProfiler::CaptureProfile to the "PROFILE" output stream.
+//
+// Example config:
+// node {
+//   calculator: "GraphProfileCalculator"
+//   output_stream: "FRAME:any_frame"
+//   output_stream: "PROFILE:graph_profile"
+// }
+//
+class GraphProfileCalculator : public Node {
+ public:
+  static constexpr Input<AnyType>::Multiple kFrameIn{"FRAME"};
+  static constexpr Output<GraphProfile> kProfileOut{"PROFILE"};
+
+  MEDIAPIPE_NODE_CONTRACT(kFrameIn, kProfileOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    auto options = cc->Options<::mediapipe::GraphProfileCalculatorOptions>();
+
+    if (prev_profile_ts_ == Timestamp::Unset() ||
+        cc->InputTimestamp() - prev_profile_ts_ >= options.profile_interval()) {
+      prev_profile_ts_ = cc->InputTimestamp();
+      GraphProfile result;
+      MP_RETURN_IF_ERROR(cc->GetProfilingContext()->CaptureProfile(&result));
+      kProfileOut(cc).Send(result);
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  Timestamp prev_profile_ts_;
+};
+
+MEDIAPIPE_REGISTER_NODE(GraphProfileCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/graph_profile_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+option objc_class_prefix = "MediaPipe";
+
+message GraphProfileCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional GraphProfileCalculatorOptions ext = 367481815;
+  }
+
+  // The interval in microseconds between successive reported GraphProfiles.
+  optional int64 profile_interval = 1 [default = 1000000];
+}

mediapipe/calculators/core/graph_profile_calculator_test.cc

@@ -0,0 +1,211 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/time/time.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_profile.pb.h"
+#include "mediapipe/framework/deps/clock.h"
+#include "mediapipe/framework/deps/message_matchers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/port/threadpool.h"
+#include "mediapipe/framework/tool/simulation_clock_executor.h"
+
+// Tests for GraphProfileCalculator.
+using testing::ElementsAre;
+
+namespace mediapipe {
+namespace {
+
+constexpr char kClockTag[] = "CLOCK";
+
+using mediapipe::Clock;
+
+// A Calculator with a fixed Process call latency.
+class SleepCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets().Tag(kClockTag).Set<std::shared_ptr<Clock>>();
+    cc->Inputs().Index(0).SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
+    cc->SetTimestampOffset(TimestampDiff(0));
+    return absl::OkStatus();
+  }
+  absl::Status Open(CalculatorContext* cc) final {
+    clock_ =
+        cc->InputSidePackets().Tag(kClockTag).Get<std::shared_ptr<Clock>>();
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    clock_->Sleep(absl::Milliseconds(5));
+    cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
+    return absl::OkStatus();
+  }
+  std::shared_ptr<::mediapipe::Clock> clock_ = nullptr;
+};
+REGISTER_CALCULATOR(SleepCalculator);
+
+// Tests showing GraphProfileCalculator reporting GraphProfile output packets.
+class GraphProfileCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUpProfileGraph() {
+    ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"(
+        input_stream: "input_packets_0"
+        node {
+          calculator: 'SleepCalculator'
+          input_side_packet: 'CLOCK:sync_clock'
+          input_stream: 'input_packets_0'
+          output_stream: 'output_packets_1'
+        }
+        node {
+          calculator: "GraphProfileCalculator"
+          options: {
+            [mediapipe.GraphProfileCalculatorOptions.ext]: {
+              profile_interval: 25000
+            }
+          }
+          input_stream: "FRAME:output_packets_1"
+          output_stream: "PROFILE:output_packets_0"
+        }
+    )",
+                                                      &graph_config_));
+  }
+
+  static Packet PacketAt(int64 ts) {
+    return Adopt(new int64(999)).At(Timestamp(ts));
+  }
+  static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); }
+  static bool IsNone(const Packet& packet) {
+    return packet.Timestamp() == Timestamp::OneOverPostStream();
+  }
+  // Return the values of the timestamps of a vector of Packets.
+  static std::vector<int64> TimestampValues(
+      const std::vector<Packet>& packets) {
+    std::vector<int64> result;
+    for (const Packet& p : packets) {
+      result.push_back(p.Timestamp().Value());
+    }
+    return result;
+  }
+
+  // Runs a CalculatorGraph with a series of packet sets.
+  // Returns a vector of packets from each graph output stream.
+  void RunGraph(const std::vector<std::vector<Packet>>& input_sets,
+                std::vector<Packet>* output_packets) {
+    // Register output packet observers.
+    tool::AddVectorSink("output_packets_0", &graph_config_, output_packets);
+
+    // Start running the graph.
+    std::shared_ptr<SimulationClockExecutor> executor(
+        new SimulationClockExecutor(3 /*num_threads*/));
+    CalculatorGraph graph;
+    MP_ASSERT_OK(graph.SetExecutor("", executor));
+    graph.profiler()->SetClock(executor->GetClock());
+    MP_ASSERT_OK(graph.Initialize(graph_config_));
+    executor->GetClock()->ThreadStart();
+    MP_ASSERT_OK(graph.StartRun({
+        {"sync_clock",
+         Adopt(new std::shared_ptr<::mediapipe::Clock>(executor->GetClock()))},
+    }));
+
+    // Send each packet to the graph in the specified order.
+    for (int t = 0; t < input_sets.size(); t++) {
+      const std::vector<Packet>& input_set = input_sets[t];
+      for (int i = 0; i < input_set.size(); i++) {
+        const Packet& packet = input_set[i];
+        if (!IsNone(packet)) {
+          MP_EXPECT_OK(graph.AddPacketToInputStream(
+              absl::StrCat("input_packets_", i), packet));
+        }
+        executor->GetClock()->Sleep(absl::Milliseconds(10));
+      }
+    }
+    MP_ASSERT_OK(graph.CloseAllInputStreams());
+    executor->GetClock()->Sleep(absl::Milliseconds(100));
+    executor->GetClock()->ThreadFinish();
+    MP_ASSERT_OK(graph.WaitUntilDone());
+  }
+
+  CalculatorGraphConfig graph_config_;
+};
+
+TEST_F(GraphProfileCalculatorTest, GraphProfile) {
+  SetUpProfileGraph();
+  auto profiler_config = graph_config_.mutable_profiler_config();
+  profiler_config->set_enable_profiler(true);
+  profiler_config->set_trace_enabled(false);
+  profiler_config->set_trace_log_disabled(true);
+  profiler_config->set_enable_stream_latency(true);
+  profiler_config->set_calculator_filter(".*Calculator");
+
+  // Run the graph with a series of packet sets.
+  std::vector<std::vector<Packet>> input_sets = {
+      {PacketAt(10000)},  //
+      {PacketAt(20000)},  //
+      {PacketAt(30000)},  //
+      {PacketAt(40000)},
+  };
+  std::vector<Packet> output_packets;
+  RunGraph(input_sets, &output_packets);
+
+  // Validate the output packets.
+  EXPECT_THAT(TimestampValues(output_packets),  //
+              ElementsAre(10000, 40000));
+
+  GraphProfile expected_profile =
+      mediapipe::ParseTextProtoOrDie<GraphProfile>(R"pb(
+        calculator_profiles {
+          name: "GraphProfileCalculator"
+          open_runtime: 0
+          process_runtime { total: 0 count: 3 }
+          process_input_latency { total: 15000 count: 3 }
+          process_output_latency { total: 15000 count: 3 }
+          input_stream_profiles {
+            name: "output_packets_1"
+            back_edge: false
+            latency { total: 0 count: 3 }
+          }
+        }
+        calculator_profiles {
+          name: "SleepCalculator"
+          open_runtime: 0
+          process_runtime { total: 15000 count: 3 }
+          process_input_latency { total: 0 count: 3 }
+          process_output_latency { total: 15000 count: 3 }
+          input_stream_profiles {
+            name: "input_packets_0"
+            back_edge: false
+            latency { total: 0 count: 3 }
+          }
+        })pb");
+
+  EXPECT_THAT(output_packets[1].Get<GraphProfile>(),
+              mediapipe::EqualsProto(expected_profile));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -29,6 +29,9 @@
 namespace mediapipe {
 namespace {
 
+constexpr char kMinuendTag[] = "MINUEND";
+constexpr char kSubtrahendTag[] = "SUBTRAHEND";
+
 // A 3x4 Matrix of random integers in [0,1000).
 const char kMatrixText[] =
     "rows: 3\n"
@@ -104,12 +107,13 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);
-  runner.MutableSidePackets()->Tag("SUBTRAHEND") = Adopt(side_matrix);
+  runner.MutableSidePackets()->Tag(kSubtrahendTag) = Adopt(side_matrix);
 
   Matrix* input_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText2, input_matrix);
-  runner.MutableInputs()->Tag("MINUEND").packets.push_back(
+  runner.MutableInputs()
-      Adopt(input_matrix).At(Timestamp(0)));
+      ->Tag(kMinuendTag)
+      .packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
 
   MP_ASSERT_OK(runner.Run());
   EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
@@ -133,12 +137,12 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
   CalculatorRunner runner(node_config);
   Matrix* side_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText, side_matrix);
-  runner.MutableSidePackets()->Tag("MINUEND") = Adopt(side_matrix);
+  runner.MutableSidePackets()->Tag(kMinuendTag) = Adopt(side_matrix);
 
   Matrix* input_matrix = new Matrix();
   MatrixFromTextProto(kMatrixText2, input_matrix);
   runner.MutableInputs()
-      ->Tag("SUBTRAHEND")
+      ->Tag(kSubtrahendTag)
       .packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
 
   MP_ASSERT_OK(runner.Run());

mediapipe/calculators/core/packet_presence_calculator.cc

@@ -17,6 +17,9 @@
 
 namespace mediapipe {
 
+constexpr char kPresenceTag[] = "PRESENCE";
+constexpr char kPacketTag[] = "PACKET";
+
 // For each non empty input packet, emits a single output packet containing a
 // boolean value "true", "false" in response to empty packets (a.k.a. timestamp
 // bound updates) This can be used to "flag" the presence of an arbitrary packet
@@ -58,8 +61,8 @@ namespace mediapipe {
 class PacketPresenceCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("PACKET").SetAny();
+    cc->Inputs().Tag(kPacketTag).SetAny();
-    cc->Outputs().Tag("PRESENCE").Set<bool>();
+    cc->Outputs().Tag(kPresenceTag).Set<bool>();
     // Process() function is invoked in response to input stream timestamp
     // bound updates.
     cc->SetProcessTimestampBounds(true);
@@ -73,8 +76,8 @@ class PacketPresenceCalculator : public CalculatorBase {
 
   absl::Status Process(CalculatorContext* cc) final {
     cc->Outputs()
-        .Tag("PRESENCE")
+        .Tag(kPresenceTag)
-        .AddPacket(MakePacket<bool>(!cc->Inputs().Tag("PACKET").IsEmpty())
+        .AddPacket(MakePacket<bool>(!cc->Inputs().Tag(kPacketTag).IsEmpty())
                        .At(cc->InputTimestamp()));
     return absl::OkStatus();
   }

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -39,6 +39,11 @@ namespace mediapipe {
 
 REGISTER_CALCULATOR(PacketResamplerCalculator);
 namespace {
+
+constexpr char kSeedTag[] = "SEED";
+constexpr char kVideoHeaderTag[] = "VIDEO_HEADER";
+constexpr char kOptionsTag[] = "OPTIONS";
+
 // Returns a TimestampDiff (assuming microseconds) corresponding to the
 // given time in seconds.
 TimestampDiff TimestampDiffFromSeconds(double seconds) {
@@ -50,16 +55,16 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) {
   const auto& resampler_options =
       cc->Options<PacketResamplerCalculatorOptions>();
-  if (cc->InputSidePackets().HasTag("OPTIONS")) {
+  if (cc->InputSidePackets().HasTag(kOptionsTag)) {
-    cc->InputSidePackets().Tag("OPTIONS").Set<CalculatorOptions>();
+    cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
   }
   CollectionItemId input_data_id = cc->Inputs().GetId("DATA", 0);
   if (!input_data_id.IsValid()) {
     input_data_id = cc->Inputs().GetId("", 0);
   }
   cc->Inputs().Get(input_data_id).SetAny();
-  if (cc->Inputs().HasTag("VIDEO_HEADER")) {
+  if (cc->Inputs().HasTag(kVideoHeaderTag)) {
-    cc->Inputs().Tag("VIDEO_HEADER").Set<VideoHeader>();
+    cc->Inputs().Tag(kVideoHeaderTag).Set<VideoHeader>();
   }
 
   CollectionItemId output_data_id = cc->Outputs().GetId("DATA", 0);
@@ -67,15 +72,15 @@ absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) {
     output_data_id = cc->Outputs().GetId("", 0);
   }
   cc->Outputs().Get(output_data_id).SetSameAs(&cc->Inputs().Get(input_data_id));
-  if (cc->Outputs().HasTag("VIDEO_HEADER")) {
+  if (cc->Outputs().HasTag(kVideoHeaderTag)) {
-    cc->Outputs().Tag("VIDEO_HEADER").Set<VideoHeader>();
+    cc->Outputs().Tag(kVideoHeaderTag).Set<VideoHeader>();
   }
 
   if (resampler_options.jitter() != 0.0) {
     RET_CHECK_GT(resampler_options.jitter(), 0.0);
     RET_CHECK_LE(resampler_options.jitter(), 1.0);
-    RET_CHECK(cc->InputSidePackets().HasTag("SEED"));
+    RET_CHECK(cc->InputSidePackets().HasTag(kSeedTag));
-    cc->InputSidePackets().Tag("SEED").Set<std::string>();
+    cc->InputSidePackets().Tag(kSeedTag).Set<std::string>();
   }
   return absl::OkStatus();
 }
@@ -143,9 +148,9 @@ absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) {
 
 absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
   if (cc->InputTimestamp() == Timestamp::PreStream() &&
-      cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") &&
+      cc->Inputs().UsesTags() && cc->Inputs().HasTag(kVideoHeaderTag) &&
-      !cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) {
+      !cc->Inputs().Tag(kVideoHeaderTag).IsEmpty()) {
-    video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get<VideoHeader>();
+    video_header_ = cc->Inputs().Tag(kVideoHeaderTag).Get<VideoHeader>();
     video_header_.frame_rate = frame_rate_;
     if (cc->Inputs().Get(input_data_id_).IsEmpty()) {
       return absl::OkStatus();
@@ -234,7 +239,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
                     "ignored, because we are adding jitter.";
   }
 
-  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
   random_ = CreateSecureRandom(seed);
   if (random_ == nullptr) {
     return absl::InvalidArgumentError(
@@ -357,7 +362,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
                     "ignored, because we are adding jitter.";
   }
 
-  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
   random_ = CreateSecureRandom(seed);
   if (random_ == nullptr) {
     return absl::InvalidArgumentError(
@@ -504,7 +509,7 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
                     "ignored, because we are adding jitter.";
   }
 
-  const auto& seed = cc->InputSidePackets().Tag("SEED").Get<std::string>();
+  const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
   random_ = CreateSecureRandom(seed);
   if (random_ == nullptr) {
     return absl::InvalidArgumentError(
@@ -635,9 +640,9 @@ absl::Status NoJitterStrategy::Process(CalculatorContext* cc) {
           base_timestamp_ +
           TimestampDiffFromSeconds(first_index / calculator_->frame_rate_);
     }
-    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) {
+    if (cc->Outputs().UsesTags() && cc->Outputs().HasTag(kVideoHeaderTag)) {
       cc->Outputs()
-          .Tag("VIDEO_HEADER")
+          .Tag(kVideoHeaderTag)
           .Add(new VideoHeader(calculator_->video_header_),
                Timestamp::PreStream());
     }

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -32,6 +32,12 @@ namespace mediapipe {
 
 using ::testing::ElementsAre;
 namespace {
+
+constexpr char kOptionsTag[] = "OPTIONS";
+constexpr char kSeedTag[] = "SEED";
+constexpr char kVideoHeaderTag[] = "VIDEO_HEADER";
+constexpr char kDataTag[] = "DATA";
+
 // A simple version of CalculatorRunner with built-in convenience
 // methods for setting inputs from a vector and checking outputs
 // against expected outputs (both timestamps and contents).
@@ -464,7 +470,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
   )pb"));
 
   for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
-    runner.MutableInputs()->Tag("DATA").packets.push_back(
+    runner.MutableInputs()->Tag(kDataTag).packets.push_back(
         Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
   }
   VideoHeader video_header_in;
@@ -474,16 +480,16 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
   video_header_in.duration = 1.0;
   video_header_in.format = ImageFormat::SRGB;
   runner.MutableInputs()
-      ->Tag("VIDEO_HEADER")
+      ->Tag(kVideoHeaderTag)
       .packets.push_back(
           Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream()));
   MP_ASSERT_OK(runner.Run());
 
-  ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size());
+  ASSERT_EQ(1, runner.Outputs().Tag(kVideoHeaderTag).packets.size());
   EXPECT_EQ(Timestamp::PreStream(),
-            runner.Outputs().Tag("VIDEO_HEADER").packets[0].Timestamp());
+            runner.Outputs().Tag(kVideoHeaderTag).packets[0].Timestamp());
   const VideoHeader& video_header_out =
-      runner.Outputs().Tag("VIDEO_HEADER").packets[0].Get<VideoHeader>();
+      runner.Outputs().Tag(kVideoHeaderTag).packets[0].Get<VideoHeader>();
   EXPECT_EQ(video_header_in.width, video_header_out.width);
   EXPECT_EQ(video_header_in.height, video_header_out.height);
   EXPECT_DOUBLE_EQ(50.0, video_header_out.frame_rate);
@@ -725,7 +731,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
               [mediapipe.PacketResamplerCalculatorOptions.ext] {
                 frame_rate: 30
               })pb"));
-    runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
+    runner.MutableSidePackets()->Tag(kOptionsTag) = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
     MP_ASSERT_OK(runner.Run());
     EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
@@ -740,7 +746,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
             frame_rate: 30
             base_timestamp: 0
           })pb"));
-    runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
+    runner.MutableSidePackets()->Tag(kOptionsTag) = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
     MP_ASSERT_OK(runner.Run());

mediapipe/calculators/core/packet_thinner_calculator_test.cc

@@ -29,6 +29,8 @@
 namespace mediapipe {
 namespace {
 
+constexpr char kPeriodTag[] = "PERIOD";
+
 // A simple version of CalculatorRunner with built-in convenience methods for
 // setting inputs from a vector and checking outputs against a vector of
 // expected outputs.
@@ -121,7 +123,7 @@ TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) {
 
   SimpleRunner runner(node);
   runner.SetInput({2, 4, 6, 8, 10, 12, 14});
-  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
   MP_ASSERT_OK(runner.Run());
 
   const std::vector<int64> expected_timestamps = {2, 8, 14};
@@ -160,7 +162,7 @@ TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) {
 
   SimpleRunner runner(node);
   runner.SetInput({2, 4, 6, 8, 10, 12, 14});
-  runner.MutableSidePackets()->Tag("PERIOD") = MakePacket<int64>(5);
+  runner.MutableSidePackets()->Tag(kPeriodTag) = MakePacket<int64>(5);
   MP_ASSERT_OK(runner.Run());
 
   const std::vector<int64> expected_timestamps = {2, 6, 10, 14};

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -39,6 +39,8 @@ using ::testing::Pair;
 using ::testing::Value;
 namespace {
 
+constexpr char kDisallowTag[] = "DISALLOW";
+
 // Returns the timestamp values for a vector of Packets.
 // TODO: puth this kind of test util in a common place.
 std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
@@ -702,14 +704,14 @@ class DroppingGateCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).SetAny();
-    cc->Inputs().Tag("DISALLOW").Set<bool>();
+    cc->Inputs().Tag(kDisallowTag).Set<bool>();
     cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) final {
     if (!cc->Inputs().Index(0).IsEmpty() &&
-        !cc->Inputs().Tag("DISALLOW").Get<bool>()) {
+        !cc->Inputs().Tag(kDisallowTag).Get<bool>()) {
       cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
     }
     return absl::OkStatus();

mediapipe/calculators/core/quantize_float_vector_calculator.cc

@@ -41,11 +41,14 @@
 //   }
 namespace mediapipe {
 
+constexpr char kEncodedTag[] = "ENCODED";
+constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
+
 class QuantizeFloatVectorCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
+    cc->Inputs().Tag(kFloatVectorTag).Set<std::vector<float>>();
-    cc->Outputs().Tag("ENCODED").Set<std::string>();
+    cc->Outputs().Tag(kEncodedTag).Set<std::string>();
     return absl::OkStatus();
   }
 
@@ -70,7 +73,7 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
 
   absl::Status Process(CalculatorContext* cc) final {
     const std::vector<float>& float_vector =
-        cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
+        cc->Inputs().Tag(kFloatVectorTag).Value().Get<std::vector<float>>();
     int feature_size = float_vector.size();
     std::string encoded_features;
     encoded_features.reserve(feature_size);
@@ -86,8 +89,10 @@ class QuantizeFloatVectorCalculator : public CalculatorBase {
           (old_value - min_quantized_value_) * (255.0 / range_));
       encoded_features += encoded;
     }
-    cc->Outputs().Tag("ENCODED").AddPacket(
+    cc->Outputs()
-        MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
+        .Tag(kEncodedTag)
+        .AddPacket(
+            MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
     return absl::OkStatus();
   }
 

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -25,6 +25,9 @@
 
 namespace mediapipe {
 
+constexpr char kEncodedTag[] = "ENCODED";
+constexpr char kFloatVectorTag[] = "FLOAT_VECTOR";
+
 TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorGraphConfig::Node node_config =
       ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
@@ -40,7 +43,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
   auto status = runner.Run();
@@ -67,7 +70,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
   auto status = runner.Run();
@@ -94,7 +97,7 @@ TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
   auto status = runner.Run();
@@ -121,11 +124,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
   CalculatorRunner runner(node_config);
   std::vector<float> empty_vector;
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
   MP_ASSERT_OK(runner.Run());
-  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  const std::vector<Packet>& outputs =
+      runner.Outputs().Tag(kEncodedTag).packets;
   EXPECT_EQ(1, outputs.size());
   EXPECT_TRUE(outputs[0].Get<std::string>().empty());
   EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
@@ -147,11 +151,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
   MP_ASSERT_OK(runner.Run());
-  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  const std::vector<Packet>& outputs =
+      runner.Outputs().Tag(kEncodedTag).packets;
   EXPECT_EQ(1, outputs.size());
   const std::string& result = outputs[0].Get<std::string>();
   ASSERT_FALSE(result.empty());
@@ -185,11 +190,12 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
   CalculatorRunner runner(node_config);
   std::vector<float> vector = {-65.0f, 65.0f};
   runner.MutableInputs()
-      ->Tag("FLOAT_VECTOR")
+      ->Tag(kFloatVectorTag)
       .packets.push_back(
           MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
   MP_ASSERT_OK(runner.Run());
-  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  const std::vector<Packet>& outputs =
+      runner.Outputs().Tag(kEncodedTag).packets;
   EXPECT_EQ(1, outputs.size());
   const std::string& result = outputs[0].Get<std::string>();
   ASSERT_FALSE(result.empty());

mediapipe/calculators/core/real_time_flow_limiter_calculator.cc

@@ -23,6 +23,9 @@
 
 namespace mediapipe {
 
+constexpr char kAllowTag[] = "ALLOW";
+constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
+
 // RealTimeFlowLimiterCalculator is used to limit the number of pipelined
 // processing operations in a section of the graph.
 //
@@ -86,11 +89,11 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
       cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i)));
     }
     cc->Inputs().Get("FINISHED", 0).SetAny();
-    if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
+    if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
-      cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set<int>();
+      cc->InputSidePackets().Tag(kMaxInFlightTag).Set<int>();
     }
-    if (cc->Outputs().HasTag("ALLOW")) {
+    if (cc->Outputs().HasTag(kAllowTag)) {
-      cc->Outputs().Tag("ALLOW").Set<bool>();
+      cc->Outputs().Tag(kAllowTag).Set<bool>();
     }
 
     cc->SetInputStreamHandler("ImmediateInputStreamHandler");
@@ -101,8 +104,8 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
   absl::Status Open(CalculatorContext* cc) final {
     finished_id_ = cc->Inputs().GetId("FINISHED", 0);
     max_in_flight_ = 1;
-    if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) {
+    if (cc->InputSidePackets().HasTag(kMaxInFlightTag)) {
-      max_in_flight_ = cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get<int>();
+      max_in_flight_ = cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>();
     }
     RET_CHECK_GE(max_in_flight_, 1);
     num_in_flight_ = 0;

mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc

@@ -33,6 +33,9 @@
 namespace mediapipe {
 
 namespace {
+
+constexpr char kFinishedTag[] = "FINISHED";
+
 // A simple Semaphore for synchronizing test threads.
 class AtomicSemaphore {
  public:
@@ -112,7 +115,7 @@ TEST(RealTimeFlowLimiterCalculator, BasicTest) {
     Timestamp timestamp =
         Timestamp((i + 1) * Timestamp::kTimestampUnitsPerSecond);
     runner.MutableInputs()
-        ->Tag("FINISHED")
+        ->Tag(kFinishedTag)
         .packets.push_back(MakePacket<bool>(true).At(timestamp));
   }
 

mediapipe/calculators/core/sequence_shift_calculator_test.cc

@@ -22,6 +22,8 @@ namespace mediapipe {
 
 namespace {
 
+constexpr char kPacketOffsetTag[] = "PACKET_OFFSET";
+
 // Adds packets containing integers equal to their original timestamp.
 void AddPackets(CalculatorRunner* runner) {
   for (int i = 0; i < 10; ++i) {
@@ -111,7 +113,7 @@ TEST(SequenceShiftCalculatorTest, SidePacketOffset) {
 
   CalculatorRunner runner(node);
   AddPackets(&runner);
-  runner.MutableSidePackets()->Tag("PACKET_OFFSET") = Adopt(new int(-2));
+  runner.MutableSidePackets()->Tag(kPacketOffsetTag) = Adopt(new int(-2));
   MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& input_packets =
       runner.MutableInputs()->Index(0).packets;

mediapipe/calculators/image/BUILD

@@ -661,3 +661,138 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
     ],
 )
+
+cc_library(
+    name = "affine_transformation",
+    hdrs = ["affine_transformation.h"],
+    deps = ["@com_google_absl//absl/status:statusor"],
+)
+
+cc_library(
+    name = "affine_transformation_runner_gl",
+    srcs = ["affine_transformation_runner_gl.cc"],
+    hdrs = ["affine_transformation_runner_gl.h"],
+    deps = [
+        ":affine_transformation",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/gpu:gl_calculator_helper",
+        "//mediapipe/gpu:gl_simple_shaders",
+        "//mediapipe/gpu:gpu_buffer",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
+        "//mediapipe/gpu:shader_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@eigen_archive//:eigen3",
+    ],
+)
+
+cc_library(
+    name = "affine_transformation_runner_opencv",
+    srcs = ["affine_transformation_runner_opencv.cc"],
+    hdrs = ["affine_transformation_runner_opencv.h"],
+    deps = [
+        ":affine_transformation",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status:statusor",
+        "@eigen_archive//:eigen3",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "warp_affine_calculator_proto",
+    srcs = ["warp_affine_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/gpu:gpu_origin_proto",
+    ],
+)
+
+cc_library(
+    name = "warp_affine_calculator",
+    srcs = ["warp_affine_calculator.cc"],
+    hdrs = ["warp_affine_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":affine_transformation",
+        ":affine_transformation_runner_opencv",
+        ":warp_affine_calculator_cc_proto",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gl_calculator_helper",
+            "//mediapipe/gpu:gpu_buffer",
+            ":affine_transformation_runner_gl",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "warp_affine_calculator_test",
+    srcs = ["warp_affine_calculator_test.cc"],
+    data = [
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/input.jpg",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
+        "//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
+    ],
+    tags = ["desktop_only_test"],
+    deps = [
+        ":affine_transformation",
+        ":warp_affine_calculator",
+        "//mediapipe/calculators/image:image_transformation_calculator",
+        "//mediapipe/calculators/tensor:image_to_tensor_converter",
+        "//mediapipe/calculators/tensor:image_to_tensor_utils",
+        "//mediapipe/calculators/util:from_image_calculator",
+        "//mediapipe/calculators/util:to_image_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
+        "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+)

mediapipe/calculators/image/affine_transformation.h

@@ -0,0 +1,55 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_
+
+#include <array>
+
+#include "absl/status/statusor.h"
+
+namespace mediapipe {
+
+class AffineTransformation {
+ public:
+  // Pixel extrapolation method.
+  // When converting image to tensor it may happen that tensor needs to read
+  // pixels outside image boundaries. Border mode helps to specify how such
+  // pixels will be calculated.
+  enum class BorderMode { kZero, kReplicate };
+
+  struct Size {
+    int width;
+    int height;
+  };
+
+  template <typename InputT, typename OutputT>
+  class Runner {
+   public:
+    virtual ~Runner() = default;
+
+    // Transforms input into output using @matrix as following:
+    //   output(x, y) = input(matrix[0] * x + matrix[1] * y + matrix[3],
+    //                        matrix[4] * x + matrix[5] * y + matrix[7])
+    // where x and y ranges are defined by @output_size.
+    virtual absl::StatusOr<OutputT> Run(const InputT& input,
+                                        const std::array<float, 16>& matrix,
+                                        const Size& output_size,
+                                        BorderMode border_mode) = 0;
+  };
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_H_

mediapipe/calculators/image/affine_transformation_runner_gl.cc

@@ -0,0 +1,354 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/affine_transformation_runner_gl.h"
+
+#include <memory>
+#include <optional>
+
+#include "Eigen/Core"
+#include "Eigen/Geometry"
+#include "Eigen/LU"
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/image/affine_transformation.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gl_simple_shaders.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
+#include "mediapipe/gpu/shader_util.h"
+
+namespace mediapipe {
+
+namespace {
+
+using mediapipe::GlCalculatorHelper;
+using mediapipe::GlhCreateProgram;
+using mediapipe::GlTexture;
+using mediapipe::GpuBuffer;
+using mediapipe::GpuOrigin;
+
+bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
+  switch (gpu_origin) {
+    case GpuOrigin::DEFAULT:
+    case GpuOrigin::CONVENTIONAL:
+#ifdef __APPLE__
+      return false;
+#else
+      return true;
+#endif  //  __APPLE__
+    case GpuOrigin::TOP_LEFT:
+      return false;
+  }
+}
+
+#ifdef __APPLE__
+#define GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED 0
+#else
+#define GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED 1
+#endif  //  __APPLE__
+
+bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) {
+  return gl_context.gl_major_version() > 3 ||
+         (gl_context.gl_major_version() == 3 &&
+          gl_context.gl_minor_version() >= 2);
+}
+
+constexpr int kAttribVertex = 0;
+constexpr int kAttribTexturePosition = 1;
+constexpr int kNumAttributes = 2;
+
+class GlTextureWarpAffineRunner
+    : public AffineTransformation::Runner<GpuBuffer,
+                                          std::unique_ptr<GpuBuffer>> {
+ public:
+  GlTextureWarpAffineRunner(std::shared_ptr<GlCalculatorHelper> gl_helper,
+                            GpuOrigin::Mode gpu_origin)
+      : gl_helper_(gl_helper), gpu_origin_(gpu_origin) {}
+  absl::Status Init() {
+    return gl_helper_->RunInGlContext([this]() -> absl::Status {
+      const GLint attr_location[kNumAttributes] = {
+          kAttribVertex,
+          kAttribTexturePosition,
+      };
+      const GLchar* attr_name[kNumAttributes] = {
+          "position",
+          "texture_coordinate",
+      };
+
+      constexpr GLchar kVertShader[] = R"(
+            in vec4 position;
+            in mediump vec4 texture_coordinate;
+            out mediump vec2 sample_coordinate;
+            uniform mat4 transform_matrix;
+
+            void main() {
+              gl_Position = position;
+              vec4 tc = transform_matrix * texture_coordinate;
+              sample_coordinate = tc.xy;
+            }
+          )";
+
+      constexpr GLchar kFragShader[] = R"(
+            DEFAULT_PRECISION(mediump, float)
+            in vec2 sample_coordinate;
+            uniform sampler2D input_texture;
+
+          #ifdef GL_ES
+            #define fragColor gl_FragColor
+          #else
+            out vec4 fragColor;
+          #endif  // defined(GL_ES);
+
+            void main() {
+              vec4 color = texture2D(input_texture, sample_coordinate);
+          #ifdef CUSTOM_ZERO_BORDER_MODE
+              float out_of_bounds =
+                  float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 ||
+                        sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0);
+              color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds);
+          #endif  // defined(CUSTOM_ZERO_BORDER_MODE)
+              fragColor = color;
+            }
+          )";
+
+      // Create program and set parameters.
+      auto create_fn = [&](const std::string& vs,
+                           const std::string& fs) -> absl::StatusOr<Program> {
+        GLuint program = 0;
+        GlhCreateProgram(vs.c_str(), fs.c_str(), kNumAttributes, &attr_name[0],
+                         attr_location, &program);
+
+        RET_CHECK(program) << "Problem initializing warp affine program.";
+        glUseProgram(program);
+        glUniform1i(glGetUniformLocation(program, "input_texture"), 1);
+        GLint matrix_id = glGetUniformLocation(program, "transform_matrix");
+        return Program{.id = program, .matrix_id = matrix_id};
+      };
+
+      const std::string vert_src =
+          absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble, kVertShader);
+
+      const std::string frag_src = absl::StrCat(
+          mediapipe::kMediaPipeFragmentShaderPreamble, kFragShader);
+
+      ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
+
+      auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
+        std::string custom_zero_border_mode_def = R"(
+          #define CUSTOM_ZERO_BORDER_MODE
+        )";
+        const std::string frag_custom_zero_src =
+            absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
+                         custom_zero_border_mode_def, kFragShader);
+        return create_fn(vert_src, frag_custom_zero_src);
+      };
+#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
+      if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
+        ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
+      }
+#else
+      ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
+#endif  // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
+
+      glGenFramebuffers(1, &framebuffer_);
+
+      // vertex storage
+      glGenBuffers(2, vbo_);
+      glGenVertexArrays(1, &vao_);
+
+      // vbo 0
+      glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
+      glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicSquareVertices),
+                   mediapipe::kBasicSquareVertices, GL_STATIC_DRAW);
+
+      // vbo 1
+      glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
+      glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicTextureVertices),
+                   mediapipe::kBasicTextureVertices, GL_STATIC_DRAW);
+
+      glBindBuffer(GL_ARRAY_BUFFER, 0);
+
+      return absl::OkStatus();
+    });
+  }
+
+  absl::StatusOr<std::unique_ptr<GpuBuffer>> Run(
+      const GpuBuffer& input, const std::array<float, 16>& matrix,
+      const AffineTransformation::Size& size,
+      AffineTransformation::BorderMode border_mode) override {
+    std::unique_ptr<GpuBuffer> gpu_buffer;
+    MP_RETURN_IF_ERROR(
+        gl_helper_->RunInGlContext([this, &input, &matrix, &size, &border_mode,
+                                    &gpu_buffer]() -> absl::Status {
+          auto input_texture = gl_helper_->CreateSourceTexture(input);
+          auto output_texture = gl_helper_->CreateDestinationTexture(
+              size.width, size.height, input.format());
+
+          MP_RETURN_IF_ERROR(
+              RunInternal(input_texture, matrix, border_mode, &output_texture));
+          gpu_buffer = output_texture.GetFrame<GpuBuffer>();
+          return absl::OkStatus();
+        }));
+
+    return gpu_buffer;
+  }
+
+  absl::Status RunInternal(const GlTexture& texture,
+                           const std::array<float, 16>& matrix,
+                           AffineTransformation::BorderMode border_mode,
+                           GlTexture* output) {
+    glDisable(GL_DEPTH_TEST);
+    glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
+    glViewport(0, 0, output->width(), output->height());
+
+    glActiveTexture(GL_TEXTURE0);
+    glBindTexture(GL_TEXTURE_2D, output->name());
+    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
+                           output->name(), 0);
+
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(texture.target(), texture.name());
+
+    // a) Filtering.
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+
+    // b) Clamping.
+    std::optional<Program> program = program_;
+    switch (border_mode) {
+      case AffineTransformation::BorderMode::kReplicate: {
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+        break;
+      }
+      case AffineTransformation::BorderMode::kZero: {
+#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
+        if (program_custom_zero_) {
+          program = program_custom_zero_;
+        } else {
+          glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
+          glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
+          glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR,
+                           std::array<float, 4>{0.0f, 0.0f, 0.0f, 0.0f}.data());
+        }
+#else
+        RET_CHECK(program_custom_zero_)
+            << "Program must have been initialized.";
+        program = program_custom_zero_;
+#endif  // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
+        break;
+      }
+    }
+    glUseProgram(program->id);
+
+    Eigen::Matrix<float, 4, 4, Eigen::RowMajor> eigen_mat(matrix.data());
+    if (IsMatrixVerticalFlipNeeded(gpu_origin_)) {
+      // @matrix describes affine transformation in terms of TOP LEFT origin, so
+      // in some cases/on some platforms an extra flipping should be done before
+      // and after.
+      const Eigen::Matrix<float, 4, 4, Eigen::RowMajor> flip_y(
+          {{1.0f, 0.0f, 0.0f, 0.0f},
+           {0.0f, -1.0f, 0.0f, 1.0f},
+           {0.0f, 0.0f, 1.0f, 0.0f},
+           {0.0f, 0.0f, 0.0f, 1.0f}});
+      eigen_mat = flip_y * eigen_mat * flip_y;
+    }
+
+    // If GL context is ES2, then GL_FALSE must be used for 'transpose'
+    // GLboolean in glUniformMatrix4fv, or else INVALID_VALUE error is reported.
+    // Hence, transposing the matrix and always passing transposed.
+    eigen_mat.transposeInPlace();
+    glUniformMatrix4fv(program->matrix_id, 1, GL_FALSE, eigen_mat.data());
+
+    // vao
+    glBindVertexArray(vao_);
+
+    // vbo 0
+    glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]);
+    glEnableVertexAttribArray(kAttribVertex);
+    glVertexAttribPointer(kAttribVertex, 2, GL_FLOAT, 0, 0, nullptr);
+
+    // vbo 1
+    glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]);
+    glEnableVertexAttribArray(kAttribTexturePosition);
+    glVertexAttribPointer(kAttribTexturePosition, 2, GL_FLOAT, 0, 0, nullptr);
+
+    // draw
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+    // Resetting to MediaPipe texture param defaults.
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+
+    glDisableVertexAttribArray(kAttribVertex);
+    glDisableVertexAttribArray(kAttribTexturePosition);
+    glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glBindVertexArray(0);
+
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(GL_TEXTURE_2D, 0);
+    glActiveTexture(GL_TEXTURE0);
+    glBindTexture(GL_TEXTURE_2D, 0);
+
+    return absl::OkStatus();
+  }
+
+  ~GlTextureWarpAffineRunner() override {
+    gl_helper_->RunInGlContext([this]() {
+      // Release OpenGL resources.
+      if (framebuffer_ != 0) glDeleteFramebuffers(1, &framebuffer_);
+      if (program_.id != 0) glDeleteProgram(program_.id);
+      if (program_custom_zero_ && program_custom_zero_->id != 0) {
+        glDeleteProgram(program_custom_zero_->id);
+      }
+      if (vao_ != 0) glDeleteVertexArrays(1, &vao_);
+      glDeleteBuffers(2, vbo_);
+    });
+  }
+
+ private:
+  struct Program {
+    GLuint id;
+    GLint matrix_id;
+  };
+  std::shared_ptr<GlCalculatorHelper> gl_helper_;
+  GpuOrigin::Mode gpu_origin_;
+  GLuint vao_ = 0;
+  GLuint vbo_[2] = {0, 0};
+  Program program_;
+  std::optional<Program> program_custom_zero_;
+  GLuint framebuffer_ = 0;
+};
+
+#undef GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
+
+}  // namespace
+
+absl::StatusOr<std::unique_ptr<
+    AffineTransformation::Runner<GpuBuffer, std::unique_ptr<GpuBuffer>>>>
+CreateAffineTransformationGlRunner(
+    std::shared_ptr<GlCalculatorHelper> gl_helper, GpuOrigin::Mode gpu_origin) {
+  auto runner =
+      absl::make_unique<GlTextureWarpAffineRunner>(gl_helper, gpu_origin);
+  MP_RETURN_IF_ERROR(runner->Init());
+  return runner;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/affine_transformation_runner_gl.h

@@ -0,0 +1,36 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_
+
+#include <memory>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/image/affine_transformation.h"
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
+
+namespace mediapipe {
+
+absl::StatusOr<std::unique_ptr<AffineTransformation::Runner<
+    mediapipe::GpuBuffer, std::unique_ptr<mediapipe::GpuBuffer>>>>
+CreateAffineTransformationGlRunner(
+    std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper,
+    mediapipe::GpuOrigin::Mode gpu_origin);
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_GL_H_

mediapipe/calculators/image/affine_transformation_runner_opencv.cc

@@ -0,0 +1,160 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/image/affine_transformation.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+namespace {
+
+cv::BorderTypes GetBorderModeForOpenCv(
+    AffineTransformation::BorderMode border_mode) {
+  switch (border_mode) {
+    case AffineTransformation::BorderMode::kZero:
+      return cv::BORDER_CONSTANT;
+    case AffineTransformation::BorderMode::kReplicate:
+      return cv::BORDER_REPLICATE;
+  }
+}
+
+class OpenCvRunner
+    : public AffineTransformation::Runner<ImageFrame, ImageFrame> {
+ public:
+  absl::StatusOr<ImageFrame> Run(
+      const ImageFrame& input, const std::array<float, 16>& matrix,
+      const AffineTransformation::Size& size,
+      AffineTransformation::BorderMode border_mode) override {
+    // OpenCV warpAffine works in absolute coordinates, so the transfom (which
+    // accepts and produces relative coordinates) should be adjusted to first
+    // normalize coordinates and then scale them.
+    // clang-format off
+    cv::Matx44f normalize_dst_coordinate({
+      1.0f / size.width, 0.0f,               0.0f, 0.0f,
+      0.0f,              1.0f / size.height, 0.0f, 0.0f,
+      0.0f,              0.0f,               1.0f, 0.0f,
+      0.0f,              0.0f,               0.0f, 1.0f});
+    cv::Matx44f scale_src_coordinate({
+      1.0f * input.Width(), 0.0f,                  0.0f, 0.0f,
+      0.0f,                 1.0f * input.Height(), 0.0f, 0.0f,
+      0.0f,                 0.0f,                  1.0f, 0.0f,
+      0.0f,                 0.0f,                  0.0f, 1.0f});
+    // clang-format on
+    cv::Matx44f adjust_dst_coordinate;
+    cv::Matx44f adjust_src_coordinate;
+    // TODO: update to always use accurate implementation.
+    constexpr bool kOpenCvCompatibility = true;
+    if (kOpenCvCompatibility) {
+      adjust_dst_coordinate = normalize_dst_coordinate;
+      adjust_src_coordinate = scale_src_coordinate;
+    } else {
+      // To do an accurate affine image transformation and make "on-cpu" and
+      // "on-gpu" calculations aligned - extra offset is required to select
+      // correct pixels.
+      //
+      // Each destination pixel corresponds to some pixels region from source
+      // image.(In case of downscaling there can be more than one pixel.) The
+      // offset for x and y is calculated in the way, so pixel in the middle of
+      // the region is selected.
+      //
+      // For simplicity sake, let's consider downscaling from 100x50 to 10x10
+      // without a rotation:
+      // 1. Each destination pixel corresponds to 10x5 region
+      //    X range: [0, .. , 9]
+      //    Y range: [0, .. , 4]
+      // 2. Considering we have __discrete__ pixels, the center of the region is
+      //    between (4, 2) and (5, 2) pixels, let's assume it's a "pixel"
+      //    (4.5, 2).
+      // 3. When using the above as an offset for every pixel select while
+      //    downscaling, resulting pixels are:
+      //      (4.5, 2), (14.5, 2), .. , (94.5, 2)
+      //      (4.5, 7), (14.5, 7), .. , (94.5, 7)
+      //      ..
+      //      (4.5, 47), (14.5, 47), .., (94.5, 47)
+      //    instead of:
+      //      (0, 0), (10, 0), .. , (90, 0)
+      //      (0, 5), (10, 7), .. , (90, 5)
+      //      ..
+      //      (0, 45), (10, 45), .., (90, 45)
+      //    The latter looks shifted.
+      //
+      // Offsets are needed, so that __discrete__ pixel at (0, 0) corresponds to
+      // the same pixel as would __non discrete__ pixel at (0.5, 0.5). Hence,
+      // transformation matrix should shift coordinates by (0.5, 0.5) as the
+      // very first step.
+      //
+      // Due to the above shift, transformed coordinates would be valid for
+      // float coordinates where pixel (0, 0) spans [0.0, 1.0) x [0.0, 1.0).
+      // T0 make it valid for __discrete__ pixels, transformation matrix should
+      // shift coordinate by (-0.5f, -0.5f) as the very last step. (E.g. if we
+      // get (0.5f, 0.5f), then it's (0, 0) __discrete__ pixel.)
+      // clang-format off
+      cv::Matx44f shift_dst({1.0f, 0.0f, 0.0f, 0.5f,
+                              0.0f, 1.0f, 0.0f, 0.5f,
+                              0.0f, 0.0f, 1.0f, 0.0f,
+                              0.0f, 0.0f, 0.0f, 1.0f});
+      cv::Matx44f shift_src({1.0f, 0.0f, 0.0f, -0.5f,
+                              0.0f, 1.0f, 0.0f, -0.5f,
+                              0.0f, 0.0f, 1.0f,  0.0f,
+                              0.0f, 0.0f, 0.0f,  1.0f});
+      // clang-format on
+      adjust_dst_coordinate = normalize_dst_coordinate * shift_dst;
+      adjust_src_coordinate = shift_src * scale_src_coordinate;
+    }
+
+    cv::Matx44f transform(matrix.data());
+    cv::Matx44f transform_absolute =
+        adjust_src_coordinate * transform * adjust_dst_coordinate;
+
+    cv::Mat in_mat = formats::MatView(&input);
+
+    cv::Mat cv_affine_transform(2, 3, CV_32F);
+    cv_affine_transform.at<float>(0, 0) = transform_absolute.val[0];
+    cv_affine_transform.at<float>(0, 1) = transform_absolute.val[1];
+    cv_affine_transform.at<float>(0, 2) = transform_absolute.val[3];
+    cv_affine_transform.at<float>(1, 0) = transform_absolute.val[4];
+    cv_affine_transform.at<float>(1, 1) = transform_absolute.val[5];
+    cv_affine_transform.at<float>(1, 2) = transform_absolute.val[7];
+
+    ImageFrame out_image(input.Format(), size.width, size.height);
+    cv::Mat out_mat = formats::MatView(&out_image);
+
+    cv::warpAffine(in_mat, out_mat, cv_affine_transform,
+                   cv::Size(out_mat.cols, out_mat.rows),
+                   /*flags=*/cv::INTER_LINEAR | cv::WARP_INVERSE_MAP,
+                   GetBorderModeForOpenCv(border_mode));
+
+    return out_image;
+  }
+};
+
+}  // namespace
+
+absl::StatusOr<
+    std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
+CreateAffineTransformationOpenCvRunner() {
+  return absl::make_unique<OpenCvRunner>();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/affine_transformation_runner_opencv.h

@@ -0,0 +1,32 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_
+
+#include <memory>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/image/affine_transformation.h"
+#include "mediapipe/framework/formats/image_frame.h"
+
+namespace mediapipe {
+
+absl::StatusOr<
+    std::unique_ptr<AffineTransformation::Runner<ImageFrame, ImageFrame>>>
+CreateAffineTransformationOpenCvRunner();
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_AFFINE_TRANSFORMATION_RUNNER_OPENCV_H_

mediapipe/calculators/image/bilateral_filter_calculator.cc

@@ -240,7 +240,7 @@ absl::Status BilateralFilterCalculator::RenderCpu(CalculatorContext* cc) {
   auto input_mat = mediapipe::formats::MatView(&input_frame);
 
   // Only 1 or 3 channel images supported by OpenCV.
-  if ((input_mat.channels() == 1 || input_mat.channels() == 3)) {
+  if (!(input_mat.channels() == 1 || input_mat.channels() == 3)) {
     return absl::InternalError(
         "CPU filtering supports only 1 or 3 channel input images.");
   }

mediapipe/calculators/image/image_clone_calculator.cc

@@ -36,7 +36,7 @@ using GpuBuffer = mediapipe::GpuBuffer;
 // stored on the target storage (CPU vs GPU) specified in the calculator option.
 //
 // The clone shares ownership of the input pixel data on the existing storage.
-// If the target storage is diffrent from the existing one, then the data is
+// If the target storage is different from the existing one, then the data is
 // further copied there.
 //
 // Example usage:

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -102,6 +102,10 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   IMAGE: ImageFrame representing the input image.
 //   IMAGE_GPU: GpuBuffer representing the input image.
 //
+//   OUTPUT_DIMENSIONS (optional): The output width and height in pixels as
+//   pair<int, int>. If set, it will override corresponding field in calculator
+//   options and input side packet.
+//
 //   ROTATION_DEGREES (optional): The counterclockwise rotation angle in
 //   degrees. This allows different rotation angles for different frames. It has
 //   to be a multiple of 90 degrees. If provided, it overrides the
@@ -221,6 +225,10 @@ absl::Status ImageTransformationCalculator::GetContract(
   }
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
+  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) {
+    cc->Inputs().Tag("OUTPUT_DIMENSIONS").Set<std::pair<int, int>>();
+  }
+
   if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
     cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
   }
@@ -329,6 +337,13 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
       !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
     flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
   }
+  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS") &&
+      !cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
+    const auto& image_size =
+        cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
+    output_width_ = image_size.first;
+    output_height_ = image_size.second;
+  }
 
   if (use_gpu_) {
 #if !MEDIAPIPE_DISABLE_GPU

mediapipe/calculators/image/scale_image_calculator.cc

@@ -262,6 +262,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
       scale_image::FindOutputDimensions(crop_width_, crop_height_,         //
                                         options_.target_width(),           //
                                         options_.target_height(),          //
+                                        options_.target_max_area(),        //
                                         options_.preserve_aspect_ratio(),  //
                                         options_.scale_to_multiple_of(),   //
                                         &output_width_, &output_height_));

mediapipe/calculators/image/scale_image_calculator.proto

@@ -28,6 +28,11 @@ message ScaleImageCalculatorOptions {
   optional int32 target_width = 1;
   optional int32 target_height = 2;
 
+  // If set, then automatically calculates a target_width and target_height that
+  // has an area below the target max area. Aspect ratio preservation cannot be
+  // disabled.
+  optional int32 target_max_area = 15;
+
   // If true, the image is scaled up or down proportionally so that it
   // fits inside the box represented by target_width and target_height.
   // Otherwise it is scaled to fit target_width and target_height

mediapipe/calculators/image/scale_image_utils.cc

@@ -92,12 +92,21 @@ absl::Status FindOutputDimensions(int input_width,             //
                                   int input_height,            //
                                   int target_width,            //
                                   int target_height,           //
+                                  int target_max_area,         //
                                   bool preserve_aspect_ratio,  //
                                   int scale_to_multiple_of,    //
                                   int* output_width, int* output_height) {
   CHECK(output_width);
   CHECK(output_height);
 
+  if (target_max_area > 0 && input_width * input_height > target_max_area) {
+    preserve_aspect_ratio = true;
+    target_height = static_cast<int>(sqrt(static_cast<double>(target_max_area) /
+                                          (static_cast<double>(input_width) /
+                                           static_cast<double>(input_height))));
+    target_width = -1;  // Resize width to preserve aspect ratio.
+  }
+
   if (preserve_aspect_ratio) {
     RET_CHECK(scale_to_multiple_of == 2)
         << "FindOutputDimensions always outputs width and height that are "
@@ -164,5 +173,17 @@ absl::Status FindOutputDimensions(int input_width,             //
       << "Unable to set output dimensions based on target dimensions.";
 }
 
+absl::Status FindOutputDimensions(int input_width,             //
+                                  int input_height,            //
+                                  int target_width,            //
+                                  int target_height,           //
+                                  bool preserve_aspect_ratio,  //
+                                  int scale_to_multiple_of,    //
+                                  int* output_width, int* output_height) {
+  return FindOutputDimensions(
+      input_width, input_height, target_width, target_height, -1,
+      preserve_aspect_ratio, scale_to_multiple_of, output_width, output_height);
+}
+
 }  // namespace scale_image
 }  // namespace mediapipe

mediapipe/calculators/image/scale_image_utils.h

@@ -34,15 +34,25 @@ absl::Status FindCropDimensions(int input_width, int input_height,    //
                                 int* crop_width, int* crop_height,    //
                                 int* col_start, int* row_start);
 
-// Given an input width and height, a target width and height, whether to
+// Given an input width and height, a target width and height or max area,
-// preserve the aspect ratio, and whether to round-down to the multiple of a
+// whether to preserve the aspect ratio, and whether to round-down to the
-// given number nearest to the targets, determine the output width and height.
+// multiple of a given number nearest to the targets, determine the output width
-// If target_width or target_height is non-positive, then they will be set to
+// and height. If target_width or target_height is non-positive, then they will
-// the input_width and input_height respectively. If scale_to_multiple_of is
+// be set to the input_width and input_height respectively. If target_area is
-// less than 1, it will be treated like 1. The output_width and
+// non-positive, then it will be ignored. If scale_to_multiple_of is less than
-// output_height will be reduced as necessary to preserve_aspect_ratio if the
+// 1, it will be treated like 1. The output_width and output_height will be
-// option is specified. If preserving the aspect ratio is desired, you must set
+// reduced as necessary to preserve_aspect_ratio if the option is specified. If
-// scale_to_multiple_of to 2.
+// preserving the aspect ratio is desired, you must set scale_to_multiple_of
+// to 2.
+absl::Status FindOutputDimensions(int input_width, int input_height,  //
+                                  int target_width,
+                                  int target_height,           //
+                                  int target_max_area,         //
+                                  bool preserve_aspect_ratio,  //
+                                  int scale_to_multiple_of,    //
+                                  int* output_width, int* output_height);
+
+// Backwards compatible helper.
 absl::Status FindOutputDimensions(int input_width, int input_height,  //
                                   int target_width,
                                   int target_height,           //

mediapipe/calculators/image/scale_image_utils_test.cc

@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
   int output_width;
   int output_height;
   // Not scale.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(100, output_height);
   // Not scale with odd input size.
-  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, -1, false, 1,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(201, output_width);
   EXPECT_EQ(101, output_height);
   // Scale down by 1/2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale up, doubling dimensions.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(400, output_width);
   EXPECT_EQ(200, output_height);
   // Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
   // visible by 2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(74, output_height);
   // Fits a 2:1 image into a 400 x 50 box.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to odd size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, -1, false, 1,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(151, output_width);
   EXPECT_EQ(101, output_height);
 }
@@ -131,18 +131,18 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
   int output_width;
   int output_height;
   // Scale width only.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, false, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(100, output_height);
   // Scale height only.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, -1, false, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(200, output_height);
   // Scale both dimensions.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, -1, false, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(200, output_height);
 }
@@ -152,41 +152,78 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
   int output_width;
   int output_height;
   // Set no targets, downscale to a multiple of 8.
-  MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, -1, false, 8,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(96, output_width);
   EXPECT_EQ(96, output_height);
   // Set width target, downscale to a multiple of 8.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, -1, false, 8,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(96, output_width);
   EXPECT_EQ(96, output_height);
   // Set height target, downscale to a multiple of 8.
-  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, -1, false, 8,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(200, output_height);
   // Set both targets, downscale to a multiple of 8.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, -1, false, 8,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(144, output_width);
   EXPECT_EQ(200, output_height);
   // Doesn't throw error if keep aspect is true and downscale multiple is 2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, -1, true, 2,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(400, output_width);
   EXPECT_EQ(200, output_height);
   // Throws error if keep aspect is true, but downscale multiple is not 2.
-  ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
+  ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, -1, true, 4,
-                                   &output_height),
+                                   &output_width, &output_height),
               testing::Not(testing::status::IsOk()));
   // Downscaling to multiple ignored if multiple is less than 2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, -1, false, 1,
-                                    &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(401, output_width);
   EXPECT_EQ(201, output_height);
 }
 
+// Tests scaling without keeping the aspect ratio fixed.
+TEST(ScaleImageUtilsTest, FindOutputDimensionsMaxArea) {
+  int output_width;
+  int output_height;
+  // Smaller area.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 9000, false, 2,
+                                    &output_width, &output_height));
+  EXPECT_NEAR(
+      200 / 100,
+      static_cast<double>(output_width) / static_cast<double>(output_height),
+      0.1f);
+  EXPECT_LE(output_width * output_height, 9000);
+  // Close to original area.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 19999, false, 2,
+                                    &output_width, &output_height));
+  EXPECT_NEAR(
+      200.0 / 100.0,
+      static_cast<double>(output_width) / static_cast<double>(output_height),
+      0.1f);
+  EXPECT_LE(output_width * output_height, 19999);
+  // Don't scale with larger area.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 20001, false, 2,
+                                    &output_width, &output_height));
+  EXPECT_EQ(200, output_width);
+  EXPECT_EQ(100, output_height);
+  // Don't scale with equal area.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, 20000, false, 2,
+                                    &output_width, &output_height));
+  EXPECT_EQ(200, output_width);
+  EXPECT_EQ(100, output_height);
+  // Don't scale at all.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, -1, false, 2,
+                                    &output_width, &output_height));
+  EXPECT_EQ(200, output_width);
+  EXPECT_EQ(100, output_height);
+}
+
 }  // namespace
 }  // namespace scale_image
 }  // namespace mediapipe

mediapipe/calculators/image/warp_affine_calculator.cc

@@ -0,0 +1,211 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/image/warp_affine_calculator.h"
+
+#include <array>
+#include <cstdint>
+#include <memory>
+
+#include "mediapipe/calculators/image/affine_transformation.h"
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/calculators/image/affine_transformation_runner_gl.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
+#include "mediapipe/calculators/image/warp_affine_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/port/ret_check.h"
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+
+namespace {
+
+AffineTransformation::BorderMode GetBorderMode(
+    mediapipe::WarpAffineCalculatorOptions::BorderMode border_mode) {
+  switch (border_mode) {
+    case mediapipe::WarpAffineCalculatorOptions::BORDER_ZERO:
+      return AffineTransformation::BorderMode::kZero;
+    case mediapipe::WarpAffineCalculatorOptions::BORDER_UNSPECIFIED:
+    case mediapipe::WarpAffineCalculatorOptions::BORDER_REPLICATE:
+      return AffineTransformation::BorderMode::kReplicate;
+  }
+}
+
+template <typename ImageT>
+class WarpAffineRunnerHolder {};
+
+template <>
+class WarpAffineRunnerHolder<ImageFrame> {
+ public:
+  using RunnerType = AffineTransformation::Runner<ImageFrame, ImageFrame>;
+  absl::Status Open(CalculatorContext* cc) { return absl::OkStatus(); }
+  absl::StatusOr<RunnerType*> GetRunner() {
+    if (!runner_) {
+      ASSIGN_OR_RETURN(runner_, CreateAffineTransformationOpenCvRunner());
+    }
+    return runner_.get();
+  }
+
+ private:
+  std::unique_ptr<RunnerType> runner_;
+};
+
+#if !MEDIAPIPE_DISABLE_GPU
+template <>
+class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
+ public:
+  using RunnerType =
+      AffineTransformation::Runner<mediapipe::GpuBuffer,
+                                   std::unique_ptr<mediapipe::GpuBuffer>>;
+  absl::Status Open(CalculatorContext* cc) {
+    gpu_origin_ =
+        cc->Options<mediapipe::WarpAffineCalculatorOptions>().gpu_origin();
+    gl_helper_ = std::make_shared<mediapipe::GlCalculatorHelper>();
+    return gl_helper_->Open(cc);
+  }
+  absl::StatusOr<RunnerType*> GetRunner() {
+    if (!runner_) {
+      ASSIGN_OR_RETURN(
+          runner_, CreateAffineTransformationGlRunner(gl_helper_, gpu_origin_));
+    }
+    return runner_.get();
+  }
+
+ private:
+  mediapipe::GpuOrigin::Mode gpu_origin_;
+  std::shared_ptr<mediapipe::GlCalculatorHelper> gl_helper_;
+  std::unique_ptr<RunnerType> runner_;
+};
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+template <>
+class WarpAffineRunnerHolder<mediapipe::Image> {
+ public:
+  absl::Status Open(CalculatorContext* cc) { return runner_.Open(cc); }
+  absl::StatusOr<
+      AffineTransformation::Runner<mediapipe::Image, mediapipe::Image>*>
+  GetRunner() {
+    return &runner_;
+  }
+
+ private:
+  class Runner : public AffineTransformation::Runner<mediapipe::Image,
+                                                     mediapipe::Image> {
+   public:
+    absl::Status Open(CalculatorContext* cc) {
+      MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
+#if !MEDIAPIPE_DISABLE_GPU
+      MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
+#endif  // !MEDIAPIPE_DISABLE_GPU
+      return absl::OkStatus();
+    }
+    absl::StatusOr<mediapipe::Image> Run(
+        const mediapipe::Image& input, const std::array<float, 16>& matrix,
+        const AffineTransformation::Size& size,
+        AffineTransformation::BorderMode border_mode) override {
+      if (input.UsesGpu()) {
+#if !MEDIAPIPE_DISABLE_GPU
+        ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
+        ASSIGN_OR_RETURN(auto result, runner->Run(input.GetGpuBuffer(), matrix,
+                                                  size, border_mode));
+        return mediapipe::Image(*result);
+#else
+        return absl::UnavailableError("GPU support is disabled");
+#endif  // !MEDIAPIPE_DISABLE_GPU
+      }
+      ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
+      const auto& frame_ptr = input.GetImageFrameSharedPtr();
+      // Wrap image into image frame.
+      const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
+                                   frame_ptr->Height(), frame_ptr->WidthStep(),
+                                   const_cast<uint8_t*>(frame_ptr->PixelData()),
+                                   [](uint8* data) {});
+      ASSIGN_OR_RETURN(auto result,
+                       runner->Run(image_frame, matrix, size, border_mode));
+      return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
+    }
+
+   private:
+    WarpAffineRunnerHolder<ImageFrame> cpu_holder_;
+#if !MEDIAPIPE_DISABLE_GPU
+    WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
+#endif  // !MEDIAPIPE_DISABLE_GPU
+  };
+
+  Runner runner_;
+};
+
+template <typename InterfaceT>
+class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
+ public:
+#if !MEDIAPIPE_DISABLE_GPU
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
+                  std::is_same_v<InterfaceT, WarpAffineCalculator>) {
+      MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+    }
+    return absl::OkStatus();
+  }
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+  absl::Status Open(CalculatorContext* cc) override { return holder_.Open(cc); }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (InterfaceT::kInImage(cc).IsEmpty() ||
+        InterfaceT::kMatrix(cc).IsEmpty() ||
+        InterfaceT::kOutputSize(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+    const std::array<float, 16>& transform = *InterfaceT::kMatrix(cc);
+    auto [out_width, out_height] = *InterfaceT::kOutputSize(cc);
+    AffineTransformation::Size output_size;
+    output_size.width = out_width;
+    output_size.height = out_height;
+    ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
+    ASSIGN_OR_RETURN(
+        auto result,
+        runner->Run(
+            *InterfaceT::kInImage(cc), transform, output_size,
+            GetBorderMode(cc->Options<mediapipe::WarpAffineCalculatorOptions>()
+                              .border_mode())));
+    InterfaceT::kOutImage(cc).Send(std::move(result));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  WarpAffineRunnerHolder<typename decltype(InterfaceT::kInImage)::PayloadT>
+      holder_;
+};
+
+}  // namespace
+
+MEDIAPIPE_NODE_IMPLEMENTATION(
+    WarpAffineCalculatorImpl<WarpAffineCalculatorCpu>);
+#if !MEDIAPIPE_DISABLE_GPU
+MEDIAPIPE_NODE_IMPLEMENTATION(
+    WarpAffineCalculatorImpl<WarpAffineCalculatorGpu>);
+#endif  // !MEDIAPIPE_DISABLE_GPU
+MEDIAPIPE_NODE_IMPLEMENTATION(WarpAffineCalculatorImpl<WarpAffineCalculator>);
+
+}  // namespace mediapipe

mediapipe/calculators/image/warp_affine_calculator.h

@@ -0,0 +1,94 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
+
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gpu_buffer.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+
+// Runs affine transformation.
+//
+// Input:
+//   IMAGE - Image/ImageFrame/GpuBuffer
+//
+//   MATRIX - std::array<float, 16>
+//     Used as following:
+//       output(x, y) = input(matrix[0] * x + matrix[1] * y + matrix[3],
+//                            matrix[4] * x + matrix[5] * y + matrix[7])
+//     where x and y ranges are defined by @OUTPUT_SIZE.
+//
+//   OUTPUT_SIZE - std::pair<int, int>
+//     Size of the output image.
+//
+// Output:
+//   IMAGE - Image/ImageFrame/GpuBuffer
+//
+//   Note:
+//   - Output image type and format are the same as the input one.
+//
+// Usage example:
+//   node {
+//     calculator: "WarpAffineCalculator(Cpu|Gpu)"
+//     input_stream: "IMAGE:image"
+//     input_stream: "MATRIX:matrix"
+//     input_stream: "OUTPUT_SIZE:size"
+//     output_stream: "IMAGE:transformed_image"
+//     options: {
+//       [mediapipe.WarpAffineCalculatorOptions.ext] {
+//         border_mode: BORDER_ZERO
+//       }
+//     }
+//   }
+template <typename ImageT>
+class WarpAffineCalculatorIntf : public mediapipe::api2::NodeIntf {
+ public:
+  static constexpr mediapipe::api2::Input<ImageT> kInImage{"IMAGE"};
+  static constexpr mediapipe::api2::Input<std::array<float, 16>> kMatrix{
+      "MATRIX"};
+  static constexpr mediapipe::api2::Input<std::pair<int, int>> kOutputSize{
+      "OUTPUT_SIZE"};
+  static constexpr mediapipe::api2::Output<ImageT> kOutImage{"IMAGE"};
+};
+
+class WarpAffineCalculatorCpu : public WarpAffineCalculatorIntf<ImageFrame> {
+ public:
+  MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorCpu, kInImage, kMatrix,
+                           kOutputSize, kOutImage);
+};
+#if !MEDIAPIPE_DISABLE_GPU
+class WarpAffineCalculatorGpu
+    : public WarpAffineCalculatorIntf<mediapipe::GpuBuffer> {
+ public:
+  MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorGpu, kInImage, kMatrix,
+                           kOutputSize, kOutImage);
+};
+#endif  // !MEDIAPIPE_DISABLE_GPU
+class WarpAffineCalculator : public WarpAffineCalculatorIntf<mediapipe::Image> {
+ public:
+  MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculator, kInImage, kMatrix, kOutputSize,
+                           kOutImage);
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_IMAGE_WARP_AFFINE_CALCULATOR_H_

mediapipe/calculators/image/warp_affine_calculator.proto

@@ -0,0 +1,46 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/gpu/gpu_origin.proto";
+
+message WarpAffineCalculatorOptions {
+  extend CalculatorOptions {
+    optional WarpAffineCalculatorOptions ext = 373693895;
+  }
+
+  // Pixel extrapolation methods. See @border_mode.
+  enum BorderMode {
+    BORDER_UNSPECIFIED = 0;
+    BORDER_ZERO = 1;
+    BORDER_REPLICATE = 2;
+  }
+
+  // Pixel extrapolation method.
+  // When converting image to tensor it may happen that tensor needs to read
+  // pixels outside image boundaries. Border mode helps to specify how such
+  // pixels will be calculated.
+  //
+  // BORDER_REPLICATE is used by default.
+  optional BorderMode border_mode = 1;
+
+  // For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs
+  // to be flipped vertically as tensors are expected to start at top.
+  // (DEFAULT or unset interpreted as CONVENTIONAL.)
+  optional GpuOrigin.Mode gpu_origin = 2;
+}

mediapipe/calculators/image/warp_affine_calculator_test.cc

@@ -0,0 +1,615 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <vector>
+
+#include "absl/flags/flag.h"
+#include "absl/memory/memory.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/image/affine_transformation.h"
+#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
+#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+cv::Mat GetRgb(absl::string_view path) {
+  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
+  cv::Mat rgb(bgr.rows, bgr.cols, CV_8UC3);
+  int from_to[] = {0, 2, 1, 1, 2, 0};
+  cv::mixChannels(&bgr, 1, &rgb, 1, from_to, 3);
+  return rgb;
+}
+
+cv::Mat GetRgba(absl::string_view path) {
+  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
+  cv::Mat rgba(bgr.rows, bgr.cols, CV_8UC4, cv::Scalar(0, 0, 0, 0));
+  int from_to[] = {0, 2, 1, 1, 2, 0};
+  cv::mixChannels(&bgr, 1, &bgr, 1, from_to, 3);
+  return bgr;
+}
+
+// Test template.
+// No processing/assertions should be done after the function is invoked.
+void RunTest(const std::string& graph_text, const std::string& tag,
+             const cv::Mat& input, cv::Mat expected_result,
+             float similarity_threshold, std::array<float, 16> matrix,
+             int out_width, int out_height,
+             absl::optional<AffineTransformation::BorderMode> border_mode) {
+  std::string border_mode_str;
+  if (border_mode) {
+    switch (*border_mode) {
+      case AffineTransformation::BorderMode::kReplicate:
+        border_mode_str = "border_mode: BORDER_REPLICATE";
+        break;
+      case AffineTransformation::BorderMode::kZero:
+        border_mode_str = "border_mode: BORDER_ZERO";
+        break;
+    }
+  }
+  auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+      absl::Substitute(graph_text, /*$0=*/border_mode_str));
+
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output_image", &graph_config, &output_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  ImageFrame input_image(
+      input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
+      input.cols, input.rows, input.step, input.data, [](uint8*) {});
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_image",
+      MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "matrix",
+      MakePacket<std::array<float, 16>>(std::move(matrix)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "output_size", MakePacket<std::pair<int, int>>(
+                         std::pair<int, int>(out_width, out_height))
+                         .At(Timestamp(0))));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  ASSERT_THAT(output_packets, testing::SizeIs(1));
+
+  // Get and process results.
+  const ImageFrame& out_frame = output_packets[0].Get<ImageFrame>();
+  cv::Mat result = formats::MatView(&out_frame);
+  double similarity =
+      1.0 - cv::norm(result, expected_result, cv::NORM_RELATIVE | cv::NORM_L2);
+  EXPECT_GE(similarity, similarity_threshold);
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  MP_ASSERT_OK(graph.CloseInputStream("input_image"));
+  MP_ASSERT_OK(graph.CloseInputStream("matrix"));
+  MP_ASSERT_OK(graph.CloseInputStream("output_size"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+enum class InputType { kImageFrame, kImage };
+
+// Similarity is checked against OpenCV results always, and due to differences
+// on how OpenCV and GL treats pixels there are two thresholds.
+// TODO: update to have just one threshold when OpenCV
+//                    implementation is updated.
+struct SimilarityConfig {
+  double threshold_on_cpu;
+  double threshold_on_gpu;
+};
+
+void RunTest(cv::Mat input, cv::Mat expected_result,
+             const SimilarityConfig& similarity, std::array<float, 16> matrix,
+             int out_width, int out_height,
+             absl::optional<AffineTransformation::BorderMode> border_mode) {
+  RunTest(R"(
+        input_stream: "input_image"
+        input_stream: "output_size"
+        input_stream: "matrix"
+        node {
+          calculator: "WarpAffineCalculatorCpu"
+          input_stream: "IMAGE:input_image"
+          input_stream: "MATRIX:matrix"
+          input_stream: "OUTPUT_SIZE:output_size"
+          output_stream: "IMAGE:output_image"
+          options {
+            [mediapipe.WarpAffineCalculatorOptions.ext] {
+              $0 # border mode
+            }
+          }
+        }
+        )",
+          "cpu", input, expected_result, similarity.threshold_on_cpu, matrix,
+          out_width, out_height, border_mode);
+
+  RunTest(R"(
+        input_stream: "input_image"
+        input_stream: "output_size"
+        input_stream: "matrix"
+        node {
+          calculator: "ToImageCalculator"
+          input_stream: "IMAGE_CPU:input_image"
+          output_stream: "IMAGE:input_image_unified"
+        }
+        node {
+          calculator: "WarpAffineCalculator"
+          input_stream: "IMAGE:input_image_unified"
+          input_stream: "MATRIX:matrix"
+          input_stream: "OUTPUT_SIZE:output_size"
+          output_stream: "IMAGE:output_image_unified"
+          options {
+            [mediapipe.WarpAffineCalculatorOptions.ext] {
+              $0 # border mode
+            }
+          }
+        }
+        node {
+          calculator: "FromImageCalculator"
+          input_stream: "IMAGE:output_image_unified"
+          output_stream: "IMAGE_CPU:output_image"
+        }
+        )",
+          "cpu_image", input, expected_result, similarity.threshold_on_cpu,
+          matrix, out_width, out_height, border_mode);
+
+  RunTest(R"(
+        input_stream: "input_image"
+        input_stream: "output_size"
+        input_stream: "matrix"
+        node {
+          calculator: "ImageFrameToGpuBufferCalculator"
+          input_stream: "input_image"
+          output_stream: "input_image_gpu"
+        }
+        node {
+          calculator: "WarpAffineCalculatorGpu"
+          input_stream: "IMAGE:input_image_gpu"
+          input_stream: "MATRIX:matrix"
+          input_stream: "OUTPUT_SIZE:output_size"
+          output_stream: "IMAGE:output_image_gpu"
+          options {
+            [mediapipe.WarpAffineCalculatorOptions.ext] {
+              $0 # border mode
+              gpu_origin: TOP_LEFT
+            }
+          }
+        }
+        node {
+          calculator: "GpuBufferToImageFrameCalculator"
+          input_stream: "output_image_gpu"
+          output_stream: "output_image"
+        }
+        )",
+          "gpu", input, expected_result, similarity.threshold_on_gpu, matrix,
+          out_width, out_height, border_mode);
+
+  RunTest(R"(
+        input_stream: "input_image"
+        input_stream: "output_size"
+        input_stream: "matrix"
+        node {
+          calculator: "ImageFrameToGpuBufferCalculator"
+          input_stream: "input_image"
+          output_stream: "input_image_gpu"
+        }
+        node {
+          calculator: "ToImageCalculator"
+          input_stream: "IMAGE_GPU:input_image_gpu"
+          output_stream: "IMAGE:input_image_unified"
+        }
+        node {
+          calculator: "WarpAffineCalculator"
+          input_stream: "IMAGE:input_image_unified"
+          input_stream: "MATRIX:matrix"
+          input_stream: "OUTPUT_SIZE:output_size"
+          output_stream: "IMAGE:output_image_unified"
+          options {
+            [mediapipe.WarpAffineCalculatorOptions.ext] {
+              $0 # border mode
+              gpu_origin: TOP_LEFT
+            }
+          }
+        }
+        node {
+          calculator: "FromImageCalculator"
+          input_stream: "IMAGE:output_image_unified"
+          output_stream: "IMAGE_GPU:output_image_gpu"
+        }
+        node {
+          calculator: "GpuBufferToImageFrameCalculator"
+          input_stream: "output_image_gpu"
+          output_stream: "output_image"
+        }
+        )",
+          "gpu_image", input, expected_result, similarity.threshold_on_gpu,
+          matrix, out_width, out_height, border_mode);
+}
+
+std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
+                                bool keep_aspect_ratio, int out_width,
+                                int out_height) {
+  std::array<float, 16> transform_mat;
+  mediapipe::RotatedRect roi_absolute =
+      mediapipe::GetRoi(input.cols, input.rows, roi);
+  mediapipe::PadRoi(out_width, out_height, keep_aspect_ratio, &roi_absolute)
+      .IgnoreError();
+  mediapipe::GetRotatedSubRectToRectTransformMatrix(
+      roi_absolute, input.cols, input.rows,
+      /*flip_horizontaly=*/false, &transform_mat);
+  return transform_mat;
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspect) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode = {};
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.82},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "medium_sub_rect_keep_aspect_border_zero.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotation) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(M_PI * 90.0f / 180.0f);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "medium_sub_rect_keep_aspect_with_rotation.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kReplicate;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.77},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectKeepAspectWithRotationBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(M_PI * 90.0f / 180.0f);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "medium_sub_rect_keep_aspect_with_rotation_border_zero.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.75},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectWithRotation) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(M_PI * -45.0f / 180.0f);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = false;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kReplicate;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.81},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, MediumSubRectWithRotationBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.65f);
+  roi.set_y_center(0.4f);
+  roi.set_width(0.5f);
+  roi.set_height(0.5f);
+  roi.set_rotation(M_PI * -45.0f / 180.0f);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "medium_sub_rect_with_rotation_border_zero.png");
+  int out_width = 256;
+  int out_height = 256;
+  bool keep_aspect_ratio = false;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.80},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRect) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/large_sub_rect.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = false;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kReplicate;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.95},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRectBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = false;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.92},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspect) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kReplicate;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(0);
+  auto input = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgb(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "large_sub_rect_keep_aspect_border_zero.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.97},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotation) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(M_PI * -15.0f / 180.0f);
+  auto input = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "large_sub_rect_keep_aspect_with_rotation.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode = {};
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.91},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, LargeSubRectKeepAspectWithRotationBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(M_PI * -15.0f / 180.0f);
+  auto input = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/"
+      "large_sub_rect_keep_aspect_with_rotation_border_zero.png");
+  int out_width = 128;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.88},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, NoOp) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.0f);
+  roi.set_height(1.0f);
+  roi.set_rotation(0);
+  auto input = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/noop_except_range.png");
+  int out_width = 64;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kReplicate;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+TEST(WarpAffineCalculatorTest, NoOpBorderZero) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.0f);
+  roi.set_height(1.0f);
+  roi.set_rotation(0);
+  auto input = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/input.jpg");
+  auto expected_output = GetRgba(
+      "/mediapipe/calculators/"
+      "tensor/testdata/image_to_tensor/noop_except_range.png");
+  int out_width = 64;
+  int out_height = 128;
+  bool keep_aspect_ratio = true;
+  std::optional<AffineTransformation::BorderMode> border_mode =
+      AffineTransformation::BorderMode::kZero;
+  RunTest(input, expected_output,
+          {.threshold_on_cpu = 0.99, .threshold_on_gpu = 0.99},
+          GetMatrix(input, roi, keep_aspect_ratio, out_width, out_height),
+          out_width, out_height, border_mode);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/BUILD

@@ -26,6 +26,11 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:private"])
 
+exports_files(
+    glob(["testdata/image_to_tensor/*"]),
+    visibility = ["//mediapipe/calculators/image:__subpackages__"],
+)
+
 selects.config_setting_group(
     name = "compute_shader_unavailable",
     match_any = [

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -87,9 +87,9 @@ using GpuBuffer = mediapipe::GpuBuffer;
 //   TENSORS - std::vector<Tensor>
 //     Vector containing a single Tensor populated with an extrated RGB image.
 //   MATRIX - std::array<float, 16> @Optional
-//     An std::array<float, 16> representing a 4x4 row-major-order matrix which
+//     An std::array<float, 16> representing a 4x4 row-major-order matrix that
-//     can be used to map a point on the output tensor to a point on the input
+//     maps a point on the input image to a point on the output tensor, and
-//     image.
+//     can be used to reverse the mapping by inverting the matrix.
 //   LETTERBOX_PADDING - std::array<float, 4> @Optional
 //     An std::array<float, 4> representing the letterbox padding from the 4
 //     sides ([left, top, right, bottom]) of the output image, normalized to

mediapipe/calculators/tensor/inference_calculator.cc

@@ -33,7 +33,7 @@ class InferenceCalculatorSelectorImpl
   absl::StatusOr<CalculatorGraphConfig> GetConfig(
       const CalculatorGraphConfig::Node& subgraph_node) {
     const auto& options =
-        Subgraph::GetOptions<::mediapipe::InferenceCalculatorOptions>(
+        Subgraph::GetOptions<mediapipe::InferenceCalculatorOptions>(
             subgraph_node);
     std::vector<absl::string_view> impls;
     const bool should_use_gpu =

mediapipe/calculators/tensor/inference_calculator.h

@@ -99,8 +99,13 @@ class InferenceCalculator : public NodeIntf {
       kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"};
   static constexpr SideInput<TfLiteModelPtr>::Optional kSideInModel{"MODEL"};
   static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
+  static constexpr SideInput<std::string>::Optional kNnApiDelegateCacheDir{
+      "NNAPI_CACHE_DIR"};
+  static constexpr SideInput<std::string>::Optional kNnApiDelegateModelToken{
+      "NNAPI_MODEL_TOKEN"};
   MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel,
-                          kOutTensors);
+                          kOutTensors, kNnApiDelegateCacheDir,
+                          kNnApiDelegateModelToken);
 
  protected:
   using TfLiteDelegatePtr =

mediapipe/calculators/tensor/inference_calculator.proto

@@ -67,9 +67,32 @@ message InferenceCalculatorOptions {
       // Only available for OpenCL delegate on Android.
       // Kernel caching will only be enabled if this path is set.
       optional string cached_kernel_path = 2;
+
+      // Encapsulated compilation/runtime tradeoffs.
+      enum InferenceUsage {
+        UNSPECIFIED = 0;
+
+        // InferenceRunner will be used only once. Therefore, it is important to
+        // minimize bootstrap time as well.
+        FAST_SINGLE_ANSWER = 1;
+
+        // Prefer maximizing the throughput. Same inference runner will be used
+        // repeatedly on different inputs.
+        SUSTAINED_SPEED = 2;
+      }
+      optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED];
     }
+
     // Android only.
-    message Nnapi {}
+    message Nnapi {
+      // Directory to store compilation cache. If unspecified, NNAPI will not
+      // try caching the compilation.
+      optional string cache_dir = 1;
+      // Unique token identifying the model. It is the caller's responsibility
+      // to ensure there is no clash of the tokens. If unspecified, NNAPI will
+      // not try caching the compilation.
+      optional string model_token = 2;
+    }
     message Xnnpack {
       // Number of threads for XNNPACK delegate. (By default, calculator tries
       // to choose optimal number of threads depending on the device.)

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -73,6 +73,7 @@ class InferenceCalculatorCpuImpl
  private:
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
   Packet<TfLiteModelPtr> model_packet_;
@@ -91,8 +92,7 @@ absl::Status InferenceCalculatorCpuImpl::UpdateContract(
 
 absl::Status InferenceCalculatorCpuImpl::Open(CalculatorContext* cc) {
   MP_RETURN_IF_ERROR(LoadModel(cc));
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+  return LoadDelegateAndAllocateTensors(cc);
-  return absl::OkStatus();
 }
 
 absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
@@ -156,11 +156,19 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 #endif  // __EMSCRIPTEN__
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorCpuImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
+  RET_CHECK_NE(
-        kTfLiteAffineQuantization);
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
-
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 
@@ -181,9 +189,21 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
     // Attempt to use NNAPI.
     // If not supported, the default CPU delegate will be created and used.
     interpreter_->SetAllowFp16PrecisionForFp32(1);
-    delegate_ = TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
+    tflite::StatefulNnApiDelegate::Options options;
-      // No need to free according to tflite::NnApiDelegate() documentation.
+    const auto& nnapi = calculator_opts.delegate().nnapi();
-    });
+    // Set up cache_dir and model_token for NNAPI compilation cache.
+    options.cache_dir =
+        nnapi.has_cache_dir() ? nnapi.cache_dir().c_str() : nullptr;
+    if (!kNnApiDelegateCacheDir(cc).IsEmpty()) {
+      options.cache_dir = kNnApiDelegateCacheDir(cc).Get().c_str();
+    }
+    options.model_token =
+        nnapi.has_model_token() ? nnapi.model_token().c_str() : nullptr;
+    if (!kNnApiDelegateModelToken(cc).IsEmpty()) {
+      options.model_token = kNnApiDelegateModelToken(cc).Get().c_str();
+    }
+    delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
+                                  [](TfLiteDelegate*) {});
     RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
                  kTfLiteOk);
     return absl::OkStatus();

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "absl/status/status.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
 #include "mediapipe/util/tflite/config.h"
 
@@ -52,6 +53,7 @@ class InferenceCalculatorGlImpl
   absl::Status WriteKernelsToFile();
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
   absl::Status InitTFLiteGPURunner(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
@@ -65,6 +67,8 @@ class InferenceCalculatorGlImpl
   bool allow_precision_loss_ = false;
   mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
       tflite_gpu_runner_api_;
+  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
+      tflite_gpu_runner_usage_;
 #endif  // MEDIAPIPE_TFLITE_GL_INFERENCE
 
 #if MEDIAPIPE_TFLITE_GPU_SUPPORTED
@@ -96,6 +100,7 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
                           options.delegate().gpu().use_advanced_gpu_api();
   allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
   tflite_gpu_runner_api_ = options.delegate().gpu().api();
+  tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
   use_kernel_caching_ = use_advanced_gpu_api_ &&
                         options.delegate().gpu().has_cached_kernel_path();
   use_gpu_delegate_ = !use_advanced_gpu_api_;
@@ -115,10 +120,11 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
   }
 
   MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this,
+  MP_RETURN_IF_ERROR(
-                                                 &cc]() -> ::mediapipe::Status {
+      gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
-    return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc) : LoadDelegate(cc);
+        return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc)
-  }));
+                                     : LoadDelegateAndAllocateTensors(cc);
+      }));
   return absl::OkStatus();
 }
 
@@ -253,9 +259,27 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
                           : tflite::gpu::InferencePriority::MAX_PRECISION;
   options.priority2 = tflite::gpu::InferencePriority::AUTO;
   options.priority3 = tflite::gpu::InferencePriority::AUTO;
-  options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+  switch (tflite_gpu_runner_usage_) {
+    case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
+        FAST_SINGLE_ANSWER: {
+      options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
+      break;
+    }
+    case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
+        SUSTAINED_SPEED: {
+      options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+      break;
+    }
+    case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::UNSPECIFIED: {
+      return absl::InternalError("inference usage need to be specified.");
+    }
+  }
   tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
   switch (tflite_gpu_runner_api_) {
+    case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
+      // Do not need to force any specific API.
+      break;
+    }
     case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::OPENGL: {
       tflite_gpu_runner_->ForceOpenGL();
       break;
@@ -264,10 +288,6 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
       tflite_gpu_runner_->ForceOpenCL();
       break;
     }
-    case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
-      // Do not need to force any specific API.
-      break;
-    }
   }
   MP_RETURN_IF_ERROR(tflite_gpu_runner_->InitializeWithModel(
       model, op_resolver, /*allow_quant_ops=*/true));
@@ -306,11 +326,19 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 #endif  // __EMSCRIPTEN__
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
+  RET_CHECK_NE(
-        kTfLiteAffineQuantization);
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
-
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -92,6 +92,7 @@ class InferenceCalculatorMetalImpl
  private:
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
   Packet<TfLiteModelPtr> model_packet_;
@@ -130,8 +131,7 @@ absl::Status InferenceCalculatorMetalImpl::Open(CalculatorContext* cc) {
 
   gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
   RET_CHECK(gpu_helper_);
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+  return LoadDelegateAndAllocateTensors(cc);
-  return absl::OkStatus();
 }
 
 absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
@@ -212,11 +212,19 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
   interpreter_->SetNumThreads(
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorMetalImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
+  RET_CHECK_NE(
-        kTfLiteAffineQuantization);
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
-
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 
@@ -236,6 +244,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
       TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), &TFLGpuDelegateDelete);
   RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
                kTfLiteOk);
+
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   // Get input image sizes.

mediapipe/calculators/tensor/tensor_converter_calculator.cc

@@ -517,8 +517,8 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
           uniform sampler2D frame;
 
           void main() {
-            $1  // flip
+            vec2 coord = $1
-            vec4 pixel = texture2D(frame, sample_coordinate);
+            vec4 pixel = texture2D(frame, coord);
             $2  // normalize [-1,1]
             fragColor.r = pixel.r;  // r channel
             $3  // g & b channels
@@ -526,8 +526,9 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
           })",
         /*$0=*/single_channel ? "vec1" : "vec4",
         /*$1=*/
-        flip_vertically_ ? "sample_coordinate.y = 1.0 - sample_coordinate.y;"
+        flip_vertically_
-                         : "",
+            ? "vec2(sample_coordinate.x, 1.0 - sample_coordinate.y);"
+            : "sample_coordinate;",
         /*$2=*/output_range_.has_value()
             ? absl::Substitute("pixel = pixel * float($0) + float($1);",
                                (output_range_->second - output_range_->first),

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -670,7 +670,8 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
         detection_boxes[box_offset + 2], detection_boxes[box_offset + 3],
         detection_scores[i], detection_classes[i], options_.flip_vertically());
     const auto& bbox = detection.location_data().relative_bounding_box();
-    if (bbox.width() < 0 || bbox.height() < 0) {
+    if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
+        std::isnan(bbox.height())) {
       // Decoded detection boxes could have negative values for width/height due
       // to model prediction. Filter out those boxes since some downstream
       // calculators may assume non-negative values. (b/171391719)

mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc

@@ -138,7 +138,6 @@ using ::tflite::gpu::gl::GlShader;
 //   }
 // }
 //
-// Currently only OpenGLES 3.1 and CPU backends supported.
 // TODO Refactor and add support for other backends/platforms.
 //
 class TensorsToSegmentationCalculator : public CalculatorBase {

mediapipe/calculators/tensorflow/BUILD

@@ -88,6 +88,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "tensor_to_vector_string_calculator_options_proto",
+    srcs = ["tensor_to_vector_string_calculator_options.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "unpack_media_sequence_calculator_proto",
     srcs = ["unpack_media_sequence_calculator.proto"],
@@ -257,6 +264,14 @@ mediapipe_cc_proto_library(
     deps = [":tensor_to_vector_float_calculator_options_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tensor_to_vector_string_calculator_options_cc_proto",
+    srcs = ["tensor_to_vector_string_calculator_options.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":tensor_to_vector_string_calculator_options_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "unpack_media_sequence_calculator_cc_proto",
     srcs = ["unpack_media_sequence_calculator.proto"],
@@ -572,9 +587,21 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
     ] + select({
         "//conditions:default": [
-            "//mediapipe/framework/port:file_helpers",
         ],
-    }),
+        "//mediapipe:android": [],
+    }) + select(
+        {
+            "//conditions:default": [
+            ],
+        },
+    ) + select(
+        {
+            "//conditions:default": [
+            ],
+            "//mediapipe:android": [
+            ],
+        },
+    ),
     alwayslink = 1,
 )
 
@@ -694,6 +721,26 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tensor_to_vector_string_calculator",
+    srcs = ["tensor_to_vector_string_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:ret_check",
+        ":tensor_to_vector_string_calculator_options_cc_proto",
+    ] + select({
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/core:framework",
+        ],
+        "//mediapipe:android": [
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "unpack_media_sequence_calculator",
     srcs = ["unpack_media_sequence_calculator.cc"],
@@ -864,6 +911,7 @@ cc_test(
         "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
@@ -1058,6 +1106,20 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "tensor_to_vector_string_calculator_test",
+    srcs = ["tensor_to_vector_string_calculator_test.cc"],
+    deps = [
+        ":tensor_to_vector_string_calculator",
+        ":tensor_to_vector_string_calculator_options_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+)
+
 cc_test(
     name = "unpack_media_sequence_calculator_test",
     srcs = ["unpack_media_sequence_calculator_test.cc"],

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -37,6 +37,7 @@ const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 const char kImageTag[] = "IMAGE";
 const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
 const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
+const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_";
 const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
 const char kBBoxTag[] = "BBOX";
 const char kKeypointsTag[] = "KEYPOINTS";
@@ -153,6 +154,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
         cc->Inputs().Tag(tag).Set<std::vector<float>>();
       }
+      if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
+        cc->Inputs().Tag(tag).Set<std::vector<std::string>>();
+      }
     }
 
     CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
@@ -231,6 +235,13 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           mpms::ClearFeatureFloats(key, sequence_.get());
           mpms::ClearFeatureTimestamp(key, sequence_.get());
         }
+        if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
+          std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
+                                           sizeof(*kBytesFeaturePrefixTag) -
+                                       1);
+          mpms::ClearFeatureBytes(key, sequence_.get());
+          mpms::ClearFeatureTimestamp(key, sequence_.get());
+        }
         if (absl::StartsWith(tag, kKeypointsTag)) {
           std::string key =
               tag.substr(sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1);
@@ -243,11 +254,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
       }
     }
 
-    if (cc->Outputs().HasTag(kSequenceExampleTag)) {
-      cc->Outputs()
-          .Tag(kSequenceExampleTag)
-          .SetNextTimestampBound(Timestamp::Max());
-    }
     return absl::OkStatus();
   }
 
@@ -305,7 +311,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
     if (cc->Outputs().HasTag(kSequenceExampleTag)) {
       cc->Outputs()
           .Tag(kSequenceExampleTag)
-          .Add(sequence_.release(), Timestamp::PostStream());
+          .Add(sequence_.release(), options.output_as_zero_timestamp()
+                                        ? Timestamp(0ll)
+                                        : Timestamp::PostStream());
     }
     sequence_.reset();
 
@@ -408,6 +416,17 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                                cc->Inputs().Tag(tag).Get<std::vector<float>>(),
                                sequence_.get());
       }
+      if (absl::StartsWith(tag, kBytesFeaturePrefixTag) &&
+          !cc->Inputs().Tag(tag).IsEmpty()) {
+        std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
+                                         sizeof(*kBytesFeaturePrefixTag) -
+                                     1);
+        mpms::AddFeatureTimestamp(key, cc->InputTimestamp().Value(),
+                                  sequence_.get());
+        mpms::AddFeatureBytes(
+            key, cc->Inputs().Tag(tag).Get<std::vector<std::string>>(),
+            sequence_.get());
+      }
       if (absl::StartsWith(tag, kBBoxTag) && !cc->Inputs().Tag(tag).IsEmpty()) {
         std::string key = "";
         if (tag != kBBoxTag) {

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto

@@ -65,4 +65,7 @@ message PackMediaSequenceCalculatorOptions {
   // If true, will return an error status if an output sequence would be too
   // many bytes to serialize.
   optional bool skip_large_sequences = 7 [default = true];
+
+  // If true/false, outputs the SequenceExample at timestamp 0/PostStream.
+  optional bool output_as_zero_timestamp = 8 [default = false];
 }

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -29,6 +29,7 @@
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/util/sequence/media_sequence.h"
 #include "tensorflow/core/example/example.pb.h"
 #include "tensorflow/core/example/feature.pb.h"
@@ -39,12 +40,33 @@ namespace {
 namespace tf = ::tensorflow;
 namespace mpms = mediapipe::mediasequence;
 
+constexpr char kBboxTag[] = "BBOX";
+constexpr char kEncodedMediaStartTimestampTag[] =
+    "ENCODED_MEDIA_START_TIMESTAMP";
+constexpr char kEncodedMediaTag[] = "ENCODED_MEDIA";
+constexpr char kClassSegmentationTag[] = "CLASS_SEGMENTATION";
+constexpr char kKeypointsTestTag[] = "KEYPOINTS_TEST";
+constexpr char kBboxPredictedTag[] = "BBOX_PREDICTED";
+constexpr char kAudioOtherTag[] = "AUDIO_OTHER";
+constexpr char kAudioTestTag[] = "AUDIO_TEST";
+constexpr char kBytesFeatureOtherTag[] = "BYTES_FEATURE_OTHER";
+constexpr char kBytesFeatureTestTag[] = "BYTES_FEATURE_TEST";
+constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
+constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
+constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
+constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
+constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
+constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
+constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
+constexpr char kImageTag[] = "IMAGE";
+
 class PackMediaSequenceCalculatorTest : public ::testing::Test {
  protected:
   void SetUpCalculator(const std::vector<std::string>& input_streams,
                        const tf::Features& features,
-                       bool output_only_if_all_present,
+                       const bool output_only_if_all_present,
-                       bool replace_instead_of_append) {
+                       const bool replace_instead_of_append,
+                       const bool output_as_zero_timestamp = false) {
     CalculatorGraphConfig::Node config;
     config.set_calculator("PackMediaSequenceCalculator");
     config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence");
@@ -57,6 +79,7 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test {
     *options->mutable_context_feature_map() = features;
     options->set_output_only_if_all_present(output_only_if_all_present);
     options->set_replace_data_instead_of_append(replace_instead_of_append);
+    options->set_output_as_zero_timestamp(output_as_zero_timestamp);
     runner_ = ::absl::make_unique<CalculatorRunner>(config);
   }
 
@@ -80,17 +103,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
   for (int i = 0; i < num_images; ++i) {
     auto image_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
-    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+    runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
         Adopt(image_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -124,17 +147,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
     auto image_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
     runner_->MutableInputs()
-        ->Tag("IMAGE_PREFIX")
+        ->Tag(kImagePrefixTag)
         .packets.push_back(Adopt(image_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -158,21 +181,21 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   for (int i = 0; i < num_timesteps; ++i) {
     auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
     runner_->MutableInputs()
-        ->Tag("FLOAT_FEATURE_TEST")
+        ->Tag(kFloatFeatureTestTag)
         .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
     vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
     runner_->MutableInputs()
-        ->Tag("FLOAT_FEATURE_OTHER")
+        ->Tag(kFloatFeatureOtherTag)
         .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -194,20 +217,65 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   }
 }
 
-TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) {
-  SetUpCalculator(
+  SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
-      {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
+                  false, true);
-      {}, false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
-  auto input_sequence = absl::make_unique<tf::SequenceExample>();
 
-  auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
+  int num_timesteps = 2;
-  runner_->MutableInputs()
+  for (int i = 0; i < num_timesteps; ++i) {
-      ->Tag("FLOAT_CONTEXT_FEATURE_TEST")
+    auto vs_ptr = ::absl::make_unique<std::vector<std::string>>(
-      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+        2, absl::StrCat("foo", 2 << i));
-  vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
+    runner_->MutableInputs()
-  runner_->MutableInputs()
+        ->Tag(kBytesFeatureTestTag)
-      ->Tag("FLOAT_CONTEXT_FEATURE_OTHER")
+        .packets.push_back(Adopt(vs_ptr.release()).At(Timestamp(i)));
-      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+    vs_ptr = ::absl::make_unique<std::vector<std::string>>(
+        2, absl::StrCat("bar", 2 << i));
+    runner_->MutableInputs()
+        ->Tag(kBytesFeatureOtherTag)
+        .packets.push_back(Adopt(vs_ptr.release()).At(Timestamp(i)));
+  }
+
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("TEST", output_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("TEST", output_sequence));
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("OTHER", output_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("OTHER", output_sequence));
+  for (int i = 0; i < num_timesteps; ++i) {
+    ASSERT_EQ(i, mpms::GetFeatureTimestampAt("TEST", output_sequence, i));
+    ASSERT_THAT(mpms::GetFeatureBytesAt("TEST", output_sequence, i),
+                ::testing::ElementsAreArray(
+                    std::vector<std::string>(2, absl::StrCat("foo", 2 << i))));
+    ASSERT_EQ(i, mpms::GetFeatureTimestampAt("OTHER", output_sequence, i));
+    ASSERT_THAT(mpms::GetFeatureBytesAt("OTHER", output_sequence, i),
+                ::testing::ElementsAreArray(
+                    std::vector<std::string>(2, absl::StrCat("bar", 2 << i))));
+  }
+}
+
+TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) {
+  SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+
+  int num_timesteps = 2;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
+    runner_->MutableInputs()
+        ->Tag("FLOAT_FEATURE_TEST")
+        .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
+  }
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
@@ -217,6 +285,32 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
   ASSERT_EQ(1, output_packets.size());
+  EXPECT_EQ(output_packets[0].Timestamp().Value(), 0ll);
+}
+
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
+  SetUpCalculator(
+      {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"},
+      {}, false, true);
+  auto input_sequence = absl::make_unique<tf::SequenceExample>();
+
+  auto vf_ptr = absl::make_unique<std::vector<float>>(2, 3);
+  runner_->MutableInputs()
+      ->Tag(kFloatContextFeatureTestTag)
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+  vf_ptr = absl::make_unique<std::vector<float>>(2, 4);
+  runner_->MutableInputs()
+      ->Tag(kFloatContextFeatureOtherTag)
+      .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream()));
+
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
+  ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
 
@@ -233,7 +327,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
   SetUpCalculator({"IMAGE:images"}, context, false, true);
 
   auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
@@ -242,13 +336,13 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   auto image_ptr =
       ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
-  runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+  runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
       Adopt(image_ptr.release()).At(Timestamp(0)));
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -281,17 +375,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
     auto flow_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
     runner_->MutableInputs()
-        ->Tag("FORWARD_FLOW_ENCODED")
+        ->Tag(kForwardFlowEncodedTag)
         .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -345,17 +439,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
     detections->push_back(detection);
 
     runner_->MutableInputs()
-        ->Tag("BBOX_PREDICTED")
+        ->Tag(kBboxPredictedTag)
         .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -424,11 +518,11 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) {
     detections->push_back(detection);
 
     runner_->MutableInputs()
-        ->Tag("BBOX_PREDICTED")
+        ->Tag(kBboxPredictedTag)
         .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   auto status = runner_->Run();
@@ -472,7 +566,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
     detections->push_back(detection);
 
     runner_->MutableInputs()
-        ->Tag("BBOX_PREDICTED")
+        ->Tag(kBboxPredictedTag)
         .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
   }
   cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
@@ -487,16 +581,16 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
   for (int i = 0; i < num_images; ++i) {
     auto image_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
-    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+    runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
         Adopt(image_ptr.release()).At(Timestamp(i)));
   }
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -538,18 +632,18 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoKeypoints) {
   absl::flat_hash_map<std::string, std::vector<std::pair<float, float>>>
       points = {{"HEAD", {{0.1, 0.2}, {0.3, 0.4}}}, {"TAIL", {{0.5, 0.6}}}};
   runner_->MutableInputs()
-      ->Tag("KEYPOINTS_TEST")
+      ->Tag(kKeypointsTestTag)
       .packets.push_back(PointToForeign(&points).At(Timestamp(0)));
   runner_->MutableInputs()
-      ->Tag("KEYPOINTS_TEST")
+      ->Tag(kKeypointsTestTag)
       .packets.push_back(PointToForeign(&points).At(Timestamp(1)));
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -589,17 +683,17 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
     detections->push_back(detection);
 
     runner_->MutableInputs()
-        ->Tag("CLASS_SEGMENTATION")
+        ->Tag(kClassSegmentationTag)
         .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -638,17 +732,17 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
     auto flow_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
     runner_->MutableInputs()
-        ->Tag("FORWARD_FLOW_ENCODED")
+        ->Tag(kForwardFlowEncodedTag)
         .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -684,11 +778,11 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
     auto flow_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_flow);
     runner_->MutableInputs()
-        ->Tag("FORWARD_FLOW_ENCODED")
+        ->Tag(kForwardFlowEncodedTag)
         .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   absl::Status status = runner_->Run();
@@ -705,13 +799,13 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingImages) {
   mpms::AddImageTimestamp(1, input_sequence.get());
   mpms::AddImageTimestamp(2, input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -731,13 +825,13 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFlowImages) {
   mpms::AddForwardFlowTimestamp(1, input_sequence.get());
   mpms::AddForwardFlowTimestamp(2, input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -768,13 +862,52 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFloatVectors) {
             mpms::GetFeatureTimestampSize("OTHER", *input_sequence));
   ASSERT_EQ(num_timesteps,
             mpms::GetFeatureFloatsSize("OTHER", *input_sequence));
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_EQ(0, mpms::GetFeatureTimestampSize("TEST", output_sequence));
+  ASSERT_EQ(0, mpms::GetFeatureFloatsSize("TEST", output_sequence));
+  ASSERT_EQ(0, mpms::GetFeatureTimestampSize("OTHER", output_sequence));
+  ASSERT_EQ(0, mpms::GetFeatureFloatsSize("OTHER", output_sequence));
+}
+
+TEST_F(PackMediaSequenceCalculatorTest, TestReplacingBytesVectors) {
+  SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
+                  false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+
+  int num_timesteps = 2;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto vs_ptr = ::absl::make_unique<std::vector<std::string>>(
+        2, absl::StrCat("foo", 2 << i));
+    mpms::AddFeatureBytes("TEST", *vs_ptr, input_sequence.get());
+    mpms::AddFeatureTimestamp("TEST", i, input_sequence.get());
+    vs_ptr = ::absl::make_unique<std::vector<std::string>>(
+        2, absl::StrCat("bar", 2 << i));
+    mpms::AddFeatureBytes("OTHER", *vs_ptr, input_sequence.get());
+    mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get());
+  }
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("TEST", *input_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("TEST", *input_sequence));
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("OTHER", *input_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureBytesSize("OTHER", *input_sequence));
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -800,7 +933,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
   for (int i = 0; i < num_images; ++i) {
     auto image_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
-    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+    runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
         Adopt(image_ptr.release()).At(Timestamp((i + 1) * 10)));
   }
 
@@ -812,11 +945,11 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
   mpms::AddBBoxTimestamp("PREFIX", 9, input_sequence.get());
   mpms::AddBBoxTimestamp("PREFIX", 22, input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   MP_ASSERT_OK(runner_->Run());
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
   ASSERT_EQ(1, output_packets.size());
   const tf::SequenceExample& output_sequence =
       output_packets[0].Get<tf::SequenceExample>();
@@ -853,7 +986,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
   for (int i = 0; i < num_images; ++i) {
     auto image_ptr =
         ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
-    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+    runner_->MutableInputs()->Tag(kImageTag).packets.push_back(
         Adopt(image_ptr.release()).At(Timestamp(i)));
   }
 
@@ -867,7 +1000,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
     Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
         .ConvertToProto(detection.mutable_location_data());
     detections->push_back(detection);
-    runner_->MutableInputs()->Tag("BBOX").packets.push_back(
+    runner_->MutableInputs()->Tag(kBboxTag).packets.push_back(
         Adopt(detections.release()).At(Timestamp(i)));
   }
 
@@ -883,7 +1016,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
     mpms::AddBBoxTrackIndex({-1}, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   // If the all the previous values aren't cleared, this assert will fail.
   MP_ASSERT_OK(runner_->Run());
@@ -899,11 +1032,11 @@ TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) {
   for (int i = 0; i < num_timesteps; ++i) {
     auto vf_ptr = ::absl::make_unique<std::vector<float>>(1000000, i);
     runner_->MutableInputs()
-        ->Tag("FLOAT_FEATURE_TEST")
+        ->Tag(kFloatFeatureTestTag)
         .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   ASSERT_FALSE(runner_->Run().ok());
 }

mediapipe/calculators/tensorflow/tensor_to_matrix_calculator_test.cc

@@ -26,6 +26,8 @@ namespace mediapipe {
 namespace tf = ::tensorflow;
 namespace {
 
+constexpr char kReferenceTag[] = "REFERENCE";
+
 constexpr char kMatrix[] = "MATRIX";
 constexpr char kTensor[] = "TENSOR";
 
@@ -68,7 +70,8 @@ class TensorToMatrixCalculatorTest : public ::testing::Test {
     if (include_rate) {
       header->set_packet_rate(1.0);
     }
-    runner_->MutableInputs()->Tag("REFERENCE").header = Adopt(header.release());
+    runner_->MutableInputs()->Tag(kReferenceTag).header =
+        Adopt(header.release());
   }
 
   std::unique_ptr<CalculatorRunner> runner_;

mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator.cc

@@ -0,0 +1,118 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Calculator converts from one-dimensional Tensor of DT_STRING to
+// vector<std::string> OR from (batched) two-dimensional Tensor of DT_STRING to
+// vector<vector<std::string>.
+
+#include "mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+
+namespace mediapipe {
+
+namespace tf = ::tensorflow;
+
+class TensorToVectorStringCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  TensorToVectorStringCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(TensorToVectorStringCalculator);
+
+absl::Status TensorToVectorStringCalculator::GetContract(
+    CalculatorContract* cc) {
+  // Start with only one input packet.
+  RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
+      << "Only one input stream is supported.";
+  cc->Inputs().Index(0).Set<tf::Tensor>(
+      // Input Tensor
+  );
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
+      << "Only one output stream is supported.";
+  const auto& options = cc->Options<TensorToVectorStringCalculatorOptions>();
+  if (options.tensor_is_2d()) {
+    RET_CHECK(!options.flatten_nd());
+    cc->Outputs().Index(0).Set<std::vector<std::vector<std::string>>>(
+        /* "Output vector<vector<std::string>>." */);
+  } else {
+    cc->Outputs().Index(0).Set<std::vector<std::string>>(
+        // Output vector<std::string>.
+    );
+  }
+  return absl::OkStatus();
+}
+
+absl::Status TensorToVectorStringCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<TensorToVectorStringCalculatorOptions>();
+
+  // Inform mediapipe that this calculator produces an output at time t for
+  // each input received at time t (i.e. this calculator does not buffer
+  // inputs). This enables mediapipe to propagate time of arrival estimates in
+  // mediapipe graphs through this calculator.
+  cc->SetOffset(/*offset=*/0);
+
+  return absl::OkStatus();
+}
+
+absl::Status TensorToVectorStringCalculator::Process(CalculatorContext* cc) {
+  const tf::Tensor& input_tensor =
+      cc->Inputs().Index(0).Value().Get<tf::Tensor>();
+  RET_CHECK(tf::DT_STRING == input_tensor.dtype())
+      << "expected DT_STRING input but got "
+      << tensorflow::DataTypeString(input_tensor.dtype());
+
+  if (options_.tensor_is_2d()) {
+    RET_CHECK(2 == input_tensor.dims())
+        << "Expected 2-dimensional Tensor, but the tensor shape is: "
+        << input_tensor.shape().DebugString();
+    auto output = absl::make_unique<std::vector<std::vector<std::string>>>(
+        input_tensor.dim_size(0),
+        std::vector<std::string>(input_tensor.dim_size(1)));
+    for (int i = 0; i < input_tensor.dim_size(0); ++i) {
+      auto& instance_output = output->at(i);
+      const auto& slice =
+          input_tensor.Slice(i, i + 1).unaligned_flat<tensorflow::tstring>();
+      for (int j = 0; j < input_tensor.dim_size(1); ++j) {
+        instance_output.at(j) = slice(j);
+      }
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  } else {
+    if (!options_.flatten_nd()) {
+      RET_CHECK(1 == input_tensor.dims())
+          << "`flatten_nd` is not set. Expected 1-dimensional Tensor, but the "
+          << "tensor shape is: " << input_tensor.shape().DebugString();
+    }
+    auto output =
+        absl::make_unique<std::vector<std::string>>(input_tensor.NumElements());
+    const auto& tensor_values = input_tensor.flat<tensorflow::tstring>();
+    for (int i = 0; i < input_tensor.NumElements(); ++i) {
+      output->at(i) = tensor_values(i);
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  }
+
+  return absl::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_options.proto

@@ -0,0 +1,33 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message TensorToVectorStringCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional TensorToVectorStringCalculatorOptions ext = 386534187;
+  }
+
+  // If true, unpack a 2d tensor (matrix) into a vector<vector<string>>. If
+  // false, convert a 1d tensor (vector) into a vector<string>.
+  optional bool tensor_is_2d = 1 [default = false];
+
+  // If true, an N-D tensor will be flattened to a vector<string>. This is
+  // exclusive with tensor_is_2d.
+  optional bool flatten_nd = 2 [default = false];
+}

mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_test.cc

@@ -0,0 +1,130 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensorflow/tensor_to_vector_string_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+namespace tf = ::tensorflow;
+
+class TensorToVectorStringCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd) {
+    CalculatorGraphConfig::Node config;
+    config.set_calculator("TensorToVectorStringCalculator");
+    config.add_input_stream("input_tensor");
+    config.add_output_stream("output_tensor");
+    auto options = config.mutable_options()->MutableExtension(
+        TensorToVectorStringCalculatorOptions::ext);
+    options->set_tensor_is_2d(tensor_is_2d);
+    options->set_flatten_nd(flatten_nd);
+    runner_ = absl::make_unique<CalculatorRunner>(config);
+  }
+
+  std::unique_ptr<CalculatorRunner> runner_;
+};
+
+TEST_F(TensorToVectorStringCalculatorTest, ConvertsToVectorFloat) {
+  SetUpRunner(false, false);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
+  auto tensor_vec = tensor->vec<tensorflow::tstring>();
+  for (int i = 0; i < 5; ++i) {
+    tensor_vec(i) = absl::StrCat("foo", i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<std::string>& output_vector =
+      output_packets[0].Get<std::vector<std::string>>();
+
+  EXPECT_EQ(5, output_vector.size());
+  for (int i = 0; i < 5; ++i) {
+    const std::string expected = absl::StrCat("foo", i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+TEST_F(TensorToVectorStringCalculatorTest, ConvertsBatchedToVectorVectorFloat) {
+  SetUpRunner(true, false);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{1, 5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
+  auto slice = tensor->Slice(0, 1).flat<tensorflow::tstring>();
+  for (int i = 0; i < 5; ++i) {
+    slice(i) = absl::StrCat("foo", i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<std::vector<std::string>>& output_vectors =
+      output_packets[0].Get<std::vector<std::vector<std::string>>>();
+  ASSERT_EQ(1, output_vectors.size());
+  const std::vector<std::string>& output_vector = output_vectors[0];
+  EXPECT_EQ(5, output_vector.size());
+  for (int i = 0; i < 5; ++i) {
+    const std::string expected = absl::StrCat("foo", i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+TEST_F(TensorToVectorStringCalculatorTest, FlattenShouldTakeAllDimensions) {
+  SetUpRunner(false, true);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{2, 2, 2});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_STRING, tensor_shape);
+  auto slice = tensor->flat<tensorflow::tstring>();
+  for (int i = 0; i < 2 * 2 * 2; ++i) {
+    slice(i) = absl::StrCat("foo", i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<std::string>& output_vector =
+      output_packets[0].Get<std::vector<std::string>>();
+  EXPECT_EQ(2 * 2 * 2, output_vector.size());
+  for (int i = 0; i < 2 * 2 * 2; ++i) {
+    const std::string expected = absl::StrCat("foo", i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -49,6 +49,11 @@ namespace tf = ::tensorflow;
 namespace mediapipe {
 
 namespace {
+
+constexpr char kRecurrentInitTensorsTag[] = "RECURRENT_INIT_TENSORS";
+constexpr char kSessionTag[] = "SESSION";
+constexpr char kSessionBundleTag[] = "SESSION_BUNDLE";
+
 // This is a simple implementation of a semaphore using standard C++ libraries.
 // It is supposed to be used only by TensorflowInferenceCalculator to throttle
 // the concurrent calls of Tensorflow Session::Run. This is useful when multiple
@@ -252,10 +257,10 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
     }
     // A mediapipe::TensorFlowSession with a model loaded and ready for use.
     // For this calculator it must include a tag_to_tensor_map.
-    cc->InputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
+    cc->InputSidePackets().Tag(kSessionTag).Set<TensorFlowSession>();
-    if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) {
+    if (cc->InputSidePackets().HasTag(kRecurrentInitTensorsTag)) {
       cc->InputSidePackets()
-          .Tag("RECURRENT_INIT_TENSORS")
+          .Tag(kRecurrentInitTensorsTag)
           .Set<std::unique_ptr<std::map<std::string, tf::Tensor>>>();
     }
     return absl::OkStatus();
@@ -265,11 +270,11 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
     std::unique_ptr<InferenceState> inference_state =
         absl::make_unique<InferenceState>();
-    if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") &&
+    if (cc->InputSidePackets().HasTag(kRecurrentInitTensorsTag) &&
-        !cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) {
+        !cc->InputSidePackets().Tag(kRecurrentInitTensorsTag).IsEmpty()) {
       std::map<std::string, tf::Tensor>* init_tensor_map;
       init_tensor_map = GetFromUniquePtr<std::map<std::string, tf::Tensor>>(
-          cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS"));
+          cc->InputSidePackets().Tag(kRecurrentInitTensorsTag));
       for (const auto& p : *init_tensor_map) {
         inference_state->input_tensor_batches_[p.first].emplace_back(p.second);
       }
@@ -280,13 +285,13 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
   absl::Status Open(CalculatorContext* cc) override {
     options_ = cc->Options<TensorFlowInferenceCalculatorOptions>();
 
-    RET_CHECK(cc->InputSidePackets().HasTag("SESSION"));
+    RET_CHECK(cc->InputSidePackets().HasTag(kSessionTag));
     session_ = cc->InputSidePackets()
-                   .Tag("SESSION")
+                   .Tag(kSessionTag)
                    .Get<TensorFlowSession>()
                    .session.get();
     tag_to_tensor_map_ = cc->InputSidePackets()
-                             .Tag("SESSION")
+                             .Tag(kSessionTag)
                              .Get<TensorFlowSession>()
                              .tag_to_tensor_map;
 

mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc

@@ -41,6 +41,11 @@ namespace mediapipe {
 namespace tf = ::tensorflow;
 
 namespace {
+
+constexpr char kMultipliedTag[] = "MULTIPLIED";
+constexpr char kBTag[] = "B";
+constexpr char kSessionTag[] = "SESSION";
+
 std::string GetGraphDefPath() {
 #ifdef __APPLE__
   char path[1024];
@@ -86,8 +91,8 @@ class TensorflowInferenceCalculatorTest : public ::testing::Test {
     MEDIAPIPE_CHECK_OK(tool::RunGenerateAndValidateTypes(
         "TensorFlowSessionFromFrozenGraphGenerator", extendable_options,
         input_side_packets, &output_side_packets));
-    runner_->MutableSidePackets()->Tag("SESSION") =
+    runner_->MutableSidePackets()->Tag(kSessionTag) =
-        output_side_packets.Tag("SESSION");
+        output_side_packets.Tag(kSessionTag);
   }
 
   Packet CreateTensorPacket(const std::vector<int32>& input, int64 time) {
@@ -140,7 +145,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetConstants) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_b =
-      runner_->Outputs().Tag("B").packets;
+      runner_->Outputs().Tag(kBTag).packets;
   ASSERT_EQ(output_packets_b.size(), 1);
   const tf::Tensor& tensor_b = output_packets_b[0].Get<tf::Tensor>();
   tf::TensorShape expected_shape({1, 3});
@@ -148,7 +153,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetConstants) {
   tf::test::ExpectTensorEqual<int32>(expected_tensor, tensor_b);
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(1, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   expected_tensor = tf::test::AsTensor<int32>({0, 0, 0}, expected_shape);
@@ -181,7 +186,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetComputed) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(1, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   tf::TensorShape expected_shape({3});
@@ -220,7 +225,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetComputed_MaxInFlight) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(1, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   tf::TensorShape expected_shape({3});
@@ -274,7 +279,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -311,7 +316,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed_MaxInFlight) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -351,7 +356,7 @@ TEST_F(TensorflowInferenceCalculatorTest,
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(3, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -392,7 +397,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetSingleBatchComputed) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -430,7 +435,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputed) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -481,7 +486,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetBatchComputed_MaxInFlight) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(5, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
@@ -528,7 +533,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   LOG(INFO) << "timestamp: " << 0;
@@ -569,7 +574,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(2, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   LOG(INFO) << "timestamp: " << 0;
@@ -662,7 +667,7 @@ TEST_F(TensorflowInferenceCalculatorTest, MissingInputFeature_Skip) {
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(0, output_packets_mult.size());
 }
 
@@ -691,7 +696,7 @@ TEST_F(TensorflowInferenceCalculatorTest,
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
-      runner_->Outputs().Tag("MULTIPLIED").packets;
+      runner_->Outputs().Tag(kMultipliedTag).packets;
   ASSERT_EQ(1, output_packets_mult.size());
   const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
   auto expected_tensor = tf::test::AsTensor<int32>({9, 12, 15});

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -47,6 +47,11 @@ namespace mediapipe {
 namespace tf = ::tensorflow;
 
 namespace {
+
+constexpr char kSessionTag[] = "SESSION";
+constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
+constexpr char kStringModelTag[] = "STRING_MODEL";
+
 // Updates the graph nodes to use the device as specified by device_id.
 void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
   for (auto& node : *graph_def->mutable_node()) {
@@ -64,30 +69,32 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
         cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
     bool has_exactly_one_model =
         !options.graph_proto_path().empty()
-            ? !(cc->InputSidePackets().HasTag("STRING_MODEL") |
+            ? !(cc->InputSidePackets().HasTag(kStringModelTag) |
-                cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"))
+                cc->InputSidePackets().HasTag(kStringModelFilePathTag))
-            : (cc->InputSidePackets().HasTag("STRING_MODEL") ^
+            : (cc->InputSidePackets().HasTag(kStringModelTag) ^
-               cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"));
+               cc->InputSidePackets().HasTag(kStringModelFilePathTag));
     RET_CHECK(has_exactly_one_model)
         << "Must have exactly one of graph_proto_path in options or "
            "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
-    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+    if (cc->InputSidePackets().HasTag(kStringModelTag)) {
       cc->InputSidePackets()
-          .Tag("STRING_MODEL")
+          .Tag(kStringModelTag)
           .Set<std::string>(
               // String model from embedded path
           );
-    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+    } else if (cc->InputSidePackets().HasTag(kStringModelFilePathTag)) {
       cc->InputSidePackets()
-          .Tag("STRING_MODEL_FILE_PATH")
+          .Tag(kStringModelFilePathTag)
           .Set<std::string>(
               // Filename of std::string model.
           );
     }
-    cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>(
+    cc->OutputSidePackets()
-        // A TensorFlow model loaded and ready for use along with
+        .Tag(kSessionTag)
-        // a map from tags to tensor names.
+        .Set<TensorFlowSession>(
-    );
+            // A TensorFlow model loaded and ready for use along with
+            // a map from tags to tensor names.
+        );
     RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
     return absl::OkStatus();
   }
@@ -111,12 +118,12 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
     session->session.reset(tf::NewSession(session_options));
 
     std::string graph_def_serialized;
-    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+    if (cc->InputSidePackets().HasTag(kStringModelTag)) {
       graph_def_serialized =
-          cc->InputSidePackets().Tag("STRING_MODEL").Get<std::string>();
+          cc->InputSidePackets().Tag(kStringModelTag).Get<std::string>();
-    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+    } else if (cc->InputSidePackets().HasTag(kStringModelFilePathTag)) {
       const std::string& frozen_graph = cc->InputSidePackets()
-                                            .Tag("STRING_MODEL_FILE_PATH")
+                                            .Tag(kStringModelFilePathTag)
                                             .Get<std::string>();
       RET_CHECK_OK(
           mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
@@ -147,7 +154,7 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
       RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
-    cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
+    cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release()));
     const uint64 end_time = absl::ToUnixMicros(clock->TimeNow());
     LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
               << " microseconds.";

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc

@@ -37,6 +37,10 @@ namespace {
 
 namespace tf = ::tensorflow;
 
+constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
+constexpr char kStringModelTag[] = "STRING_MODEL";
+constexpr char kSessionTag[] = "SESSION";
+
 std::string GetGraphDefPath() {
   return mediapipe::file::JoinPath("./",
                                    "mediapipe/calculators/tensorflow/"
@@ -112,7 +116,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
 
   MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   VerifySignatureMap(session);
 }
 
@@ -190,12 +194,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
   std::string serialized_graph_contents;
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
-  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+  runner.MutableSidePackets()->Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
   MP_ASSERT_OK(runner.Run());
 
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   VerifySignatureMap(session);
 }
 
@@ -213,12 +217,12 @@ TEST_F(
           }
         })",
                                            calculator_options_->DebugString()));
-  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+  runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   MP_ASSERT_OK(runner.Run());
 
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   VerifySignatureMap(session);
 }
 
@@ -234,7 +238,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
           }
         })",
                                            calculator_options_->DebugString()));
-  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+  runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   auto run_status = runner.Run();
   EXPECT_THAT(
@@ -255,12 +259,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
           }
         })",
                                            calculator_options_->DebugString()));
-  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+  runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   std::string serialized_graph_contents;
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
-  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+  runner.MutableSidePackets()->Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
   auto run_status = runner.Run();
   EXPECT_THAT(
@@ -282,12 +286,12 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
           }
         })",
                                            calculator_options_->DebugString()));
-  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+  runner.MutableSidePackets()->Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   std::string serialized_graph_contents;
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
-  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+  runner.MutableSidePackets()->Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
   auto run_status = runner.Run();
   EXPECT_THAT(
@@ -310,7 +314,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
   MP_ASSERT_OK(runner.Run());
 
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   VerifySignatureMap(session);
 }
 

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc

@@ -43,6 +43,11 @@ namespace mediapipe {
 namespace tf = ::tensorflow;
 
 namespace {
+
+constexpr char kSessionTag[] = "SESSION";
+constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
+constexpr char kStringModelTag[] = "STRING_MODEL";
+
 // Updates the graph nodes to use the device as specified by device_id.
 void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
   for (auto& node : *graph_def->mutable_node()) {
@@ -64,28 +69,29 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
         TensorFlowSessionFromFrozenGraphGeneratorOptions::ext);
     bool has_exactly_one_model =
         !options.graph_proto_path().empty()
-            ? !(input_side_packets->HasTag("STRING_MODEL") |
+            ? !(input_side_packets->HasTag(kStringModelTag) |
-                input_side_packets->HasTag("STRING_MODEL_FILE_PATH"))
+                input_side_packets->HasTag(kStringModelFilePathTag))
-            : (input_side_packets->HasTag("STRING_MODEL") ^
+            : (input_side_packets->HasTag(kStringModelTag) ^
-               input_side_packets->HasTag("STRING_MODEL_FILE_PATH"));
+               input_side_packets->HasTag(kStringModelFilePathTag));
     RET_CHECK(has_exactly_one_model)
         << "Must have exactly one of graph_proto_path in options or "
            "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
-    if (input_side_packets->HasTag("STRING_MODEL")) {
+    if (input_side_packets->HasTag(kStringModelTag)) {
-      input_side_packets->Tag("STRING_MODEL")
+      input_side_packets->Tag(kStringModelTag)
           .Set<std::string>(
               // String model from embedded path
           );
-    } else if (input_side_packets->HasTag("STRING_MODEL_FILE_PATH")) {
+    } else if (input_side_packets->HasTag(kStringModelFilePathTag)) {
-      input_side_packets->Tag("STRING_MODEL_FILE_PATH")
+      input_side_packets->Tag(kStringModelFilePathTag)
           .Set<std::string>(
               // Filename of std::string model.
           );
     }
-    output_side_packets->Tag("SESSION").Set<TensorFlowSession>(
+    output_side_packets->Tag(kSessionTag)
-        // A TensorFlow model loaded and ready for use along with
+        .Set<TensorFlowSession>(
-        // a map from tags to tensor names.
+            // A TensorFlow model loaded and ready for use along with
-    );
+            // a map from tags to tensor names.
+        );
     RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
     return absl::OkStatus();
   }
@@ -112,12 +118,12 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
     session->session.reset(tf::NewSession(session_options));
 
     std::string graph_def_serialized;
-    if (input_side_packets.HasTag("STRING_MODEL")) {
+    if (input_side_packets.HasTag(kStringModelTag)) {
       graph_def_serialized =
-          input_side_packets.Tag("STRING_MODEL").Get<std::string>();
+          input_side_packets.Tag(kStringModelTag).Get<std::string>();
-    } else if (input_side_packets.HasTag("STRING_MODEL_FILE_PATH")) {
+    } else if (input_side_packets.HasTag(kStringModelFilePathTag)) {
       const std::string& frozen_graph =
-          input_side_packets.Tag("STRING_MODEL_FILE_PATH").Get<std::string>();
+          input_side_packets.Tag(kStringModelFilePathTag).Get<std::string>();
       RET_CHECK_OK(
           mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
     } else {
@@ -147,7 +153,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
       RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString();
     }
 
-    output_side_packets->Tag("SESSION") = Adopt(session.release());
+    output_side_packets->Tag(kSessionTag) = Adopt(session.release());
     const uint64 end_time = absl::ToUnixMicros(clock->TimeNow());
     LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
               << " microseconds.";

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc

@@ -37,6 +37,10 @@ namespace {
 
 namespace tf = ::tensorflow;
 
+constexpr char kStringModelFilePathTag[] = "STRING_MODEL_FILE_PATH";
+constexpr char kStringModelTag[] = "STRING_MODEL";
+constexpr char kSessionTag[] = "SESSION";
+
 std::string GetGraphDefPath() {
   return mediapipe::file::JoinPath("./",
                                    "mediapipe/calculators/tensorflow/"
@@ -72,7 +76,7 @@ class TensorFlowSessionFromFrozenGraphGeneratorTest : public ::testing::Test {
 
   void VerifySignatureMap(PacketSet* output_side_packets) {
     const TensorFlowSession& session =
-        output_side_packets->Tag("SESSION").Get<TensorFlowSession>();
+        output_side_packets->Tag(kSessionTag).Get<TensorFlowSession>();
     // Session must be set.
     ASSERT_NE(session.session, nullptr);
 
@@ -179,7 +183,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
   generator_options_->clear_graph_proto_path();
-  input_side_packets.Tag("STRING_MODEL") =
+  input_side_packets.Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
   absl::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
@@ -196,7 +200,7 @@ TEST_F(
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).value());
   generator_options_->clear_graph_proto_path();
-  input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
+  input_side_packets.Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   absl::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
@@ -211,7 +215,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
       tool::CreateTagMap({"STRING_MODEL_FILE_PATH:model_path"}).value());
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).value());
-  input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
+  input_side_packets.Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   absl::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
@@ -233,9 +237,9 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   std::string serialized_graph_contents;
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
-  input_side_packets.Tag("STRING_MODEL") =
+  input_side_packets.Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
-  input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
+  input_side_packets.Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
 
   absl::Status run_status = tool::RunGenerateAndValidateTypes(
@@ -258,9 +262,9 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   std::string serialized_graph_contents;
   MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
                                             &serialized_graph_contents));
-  input_side_packets.Tag("STRING_MODEL") =
+  input_side_packets.Tag(kStringModelTag) =
       Adopt(new std::string(serialized_graph_contents));
-  input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
+  input_side_packets.Tag(kStringModelFilePathTag) =
       Adopt(new std::string(GetGraphDefPath()));
   generator_options_->clear_graph_proto_path();
 

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -31,6 +31,9 @@
 namespace mediapipe {
 
 namespace {
+
+constexpr char kSessionTag[] = "SESSION";
+
 static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 
 // Given the path to a directory containing multiple tensorflow saved models
@@ -108,7 +111,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
       cc->InputSidePackets().Tag(kStringSavedModelPath).Set<std::string>();
     }
     // A TensorFlow model loaded and ready for use along with tensor
-    cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>();
+    cc->OutputSidePackets().Tag(kSessionTag).Set<TensorFlowSession>();
     return absl::OkStatus();
   }
 
@@ -160,7 +163,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
           output_signature.first, options)] = output_signature.second.name();
     }
 
-    cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
+    cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release()));
     return absl::OkStatus();
   }
 

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc

@@ -35,6 +35,9 @@ namespace {
 
 namespace tf = ::tensorflow;
 
+constexpr char kStringSavedModelPathTag[] = "STRING_SAVED_MODEL_PATH";
+constexpr char kSessionTag[] = "SESSION";
+
 std::string GetSavedModelDir() {
   std::string out_path =
       file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/",
@@ -79,7 +82,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
                                            options_->DebugString()));
   MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 
@@ -119,11 +122,11 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
           }
         })",
                                            options_->DebugString()));
-  runner.MutableSidePackets()->Tag("STRING_SAVED_MODEL_PATH") =
+  runner.MutableSidePackets()->Tag(kStringSavedModelPathTag) =
       MakePacket<std::string>(GetSavedModelDir());
   MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 }
@@ -201,7 +204,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
                                            options_->DebugString()));
   MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 }
@@ -224,7 +227,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
                                            options_->DebugString()));
   MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
-      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+      runner.OutputSidePackets().Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
   std::vector<tensorflow::DeviceAttributes> devices;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -33,6 +33,9 @@
 namespace mediapipe {
 
 namespace {
+
+constexpr char kSessionTag[] = "SESSION";
+
 static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 
 // Given the path to a directory containing multiple tensorflow saved models
@@ -100,7 +103,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
       input_side_packets->Tag(kStringSavedModelPath).Set<std::string>();
     }
     // A TensorFlow model loaded and ready for use along with tensor
-    output_side_packets->Tag("SESSION").Set<TensorFlowSession>();
+    output_side_packets->Tag(kSessionTag).Set<TensorFlowSession>();
     return absl::OkStatus();
   }
 
@@ -153,7 +156,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
           output_signature.first, options)] = output_signature.second.name();
     }
 
-    output_side_packets->Tag("SESSION") = Adopt(session.release());
+    output_side_packets->Tag(kSessionTag) = Adopt(session.release());
     return absl::OkStatus();
   }
 };

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc

@@ -34,6 +34,9 @@ namespace {
 
 namespace tf = ::tensorflow;
 
+constexpr char kStringSavedModelPathTag[] = "STRING_SAVED_MODEL_PATH";
+constexpr char kSessionTag[] = "SESSION";
+
 std::string GetSavedModelDir() {
   std::string out_path =
       file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/",
@@ -75,7 +78,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
       input_side_packets, &output_side_packets);
   MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
-      output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
+      output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 
@@ -107,7 +110,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
   generator_options_->clear_saved_model_path();
   PacketSet input_side_packets(
       tool::CreateTagMap({"STRING_SAVED_MODEL_PATH:saved_model_dir"}).value());
-  input_side_packets.Tag("STRING_SAVED_MODEL_PATH") =
+  input_side_packets.Tag(kStringSavedModelPathTag) =
       Adopt(new std::string(GetSavedModelDir()));
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).value());
@@ -116,7 +119,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
       input_side_packets, &output_side_packets);
   MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
-      output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
+      output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 }
@@ -192,7 +195,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
       input_side_packets, &output_side_packets);
   MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
-      output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
+      output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
 }
@@ -213,7 +216,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
       input_side_packets, &output_side_packets);
   MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
-      output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
+      output_side_packets.Tag(kSessionTag).Get<TensorFlowSession>();
   // Session must be set.
   ASSERT_NE(session.session, nullptr);
   std::vector<tensorflow::DeviceAttributes> devices;

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc

@@ -33,6 +33,33 @@ namespace {
 namespace tf = ::tensorflow;
 namespace mpms = mediapipe::mediasequence;
 
+constexpr char kImageFrameRateTag[] = "IMAGE_FRAME_RATE";
+constexpr char kEncodedMediaStartTimestampTag[] =
+    "ENCODED_MEDIA_START_TIMESTAMP";
+constexpr char kEncodedMediaTag[] = "ENCODED_MEDIA";
+constexpr char kResamplerOptionsTag[] = "RESAMPLER_OPTIONS";
+constexpr char kSandboxedDecoderOptionsTag[] = "SANDBOXED_DECODER_OPTIONS";
+constexpr char kDecoderOptionsTag[] = "DECODER_OPTIONS";
+constexpr char kAudioDecoderOptionsTag[] = "AUDIO_DECODER_OPTIONS";
+constexpr char kDataPathTag[] = "DATA_PATH";
+constexpr char kDatasetRootTag[] = "DATASET_ROOT";
+constexpr char kMediaIdTag[] = "MEDIA_ID";
+constexpr char kFloatFeatureFdenseMaxTag[] = "FLOAT_FEATURE_FDENSE_MAX";
+constexpr char kFloatFeatureFdenseAvgTag[] = "FLOAT_FEATURE_FDENSE_AVG";
+constexpr char kAudioOtherTag[] = "AUDIO_OTHER";
+constexpr char kAudioTestTag[] = "AUDIO_TEST";
+constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
+constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
+constexpr char kBboxPrefixTag[] = "BBOX_PREFIX";
+constexpr char kKeypointsTag[] = "KEYPOINTS";
+constexpr char kBboxTag[] = "BBOX";
+constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
+constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
+constexpr char kImageTag[] = "IMAGE";
+constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
+constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
+constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
+
 class UnpackMediaSequenceCalculatorTest : public ::testing::Test {
  protected:
   void SetUpCalculator(const std::vector<std::string>& output_streams,
@@ -95,13 +122,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneImage) {
     mpms::AddImageEncoded(test_image_string, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("IMAGE").packets;
+      runner_->Outputs().Tag(kImageTag).packets;
   ASSERT_EQ(num_images, output_packets.size());
 
   for (int i = 0; i < num_images; ++i) {
@@ -124,13 +151,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoImages) {
     mpms::AddImageEncoded(test_image_string, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("IMAGE").packets;
+      runner_->Outputs().Tag(kImageTag).packets;
   ASSERT_EQ(num_images, output_packets.size());
 
   for (int i = 0; i < num_images; ++i) {
@@ -154,13 +181,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPrefixedImages) {
     mpms::AddImageEncoded(prefix, test_image_string, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("IMAGE_PREFIX").packets;
+      runner_->Outputs().Tag(kImagePrefixTag).packets;
   ASSERT_EQ(num_images, output_packets.size());
 
   for (int i = 0; i < num_images; ++i) {
@@ -182,12 +209,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneForwardFlowImage) {
     mpms::AddForwardFlowEncoded(test_image_string, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
+      runner_->Outputs().Tag(kForwardFlowEncodedTag).packets;
   ASSERT_EQ(num_forward_flow_images, output_packets.size());
 
   for (int i = 0; i < num_forward_flow_images; ++i) {
@@ -211,12 +238,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoForwardFlowImages) {
     mpms::AddForwardFlowEncoded(test_image_strings[i], input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
+      runner_->Outputs().Tag(kForwardFlowEncodedTag).packets;
   ASSERT_EQ(num_forward_flow_images, output_packets.size());
 
   for (int i = 0; i < num_forward_flow_images; ++i) {
@@ -240,13 +267,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksBBoxes) {
     mpms::AddBBoxTimestamp(i, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("BBOX").packets;
+      runner_->Outputs().Tag(kBboxTag).packets;
   ASSERT_EQ(bboxes.size(), output_packets.size());
 
   for (int i = 0; i < bboxes.size(); ++i) {
@@ -274,13 +301,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPrefixedBBoxes) {
     mpms::AddBBoxTimestamp(prefix, i, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("BBOX_PREFIX").packets;
+      runner_->Outputs().Tag(kBboxPrefixTag).packets;
   ASSERT_EQ(bboxes.size(), output_packets.size());
 
   for (int i = 0; i < bboxes.size(); ++i) {
@@ -306,13 +333,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) {
     mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("FLOAT_FEATURE_TEST").packets;
+      runner_->Outputs().Tag(kFloatFeatureTestTag).packets;
   ASSERT_EQ(num_float_lists, output_packets.size());
 
   for (int i = 0; i < num_float_lists; ++i) {
@@ -322,7 +349,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) {
   }
 
   const std::vector<Packet>& output_packets_other =
-      runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets;
+      runner_->Outputs().Tag(kFloatFeatureOtherTag).packets;
   ASSERT_EQ(num_float_lists, output_packets_other.size());
 
   for (int i = 0; i < num_float_lists; ++i) {
@@ -352,12 +379,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) {
     mpms::AddFeatureTimestamp("OTHER", i + 5, input_sequence.get());
   }
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("IMAGE").packets;
+      runner_->Outputs().Tag(kImageTag).packets;
   ASSERT_EQ(num_images, output_packets.size());
 
   for (int i = 0; i < num_images; ++i) {
@@ -366,7 +393,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) {
   }
 
   const std::vector<Packet>& output_packets_other =
-      runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets;
+      runner_->Outputs().Tag(kFloatFeatureOtherTag).packets;
   ASSERT_EQ(num_float_lists, output_packets_other.size());
 
   for (int i = 0; i < num_float_lists; ++i) {
@@ -389,12 +416,12 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) {
   mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
                             input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& fdense_avg_packets =
-      runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_AVG").packets;
+      runner_->Outputs().Tag(kFloatFeatureFdenseAvgTag).packets;
   ASSERT_EQ(fdense_avg_packets.size(), 1);
   const auto& fdense_avg_vector =
       fdense_avg_packets[0].Get<std::vector<float>>();
@@ -403,7 +430,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) {
               ::testing::Eq(Timestamp::PostStream()));
 
   const std::vector<Packet>& fdense_max_packets =
-      runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets;
+      runner_->Outputs().Tag(kFloatFeatureFdenseMaxTag).packets;
   ASSERT_EQ(fdense_max_packets.size(), 1);
   const auto& fdense_max_vector =
       fdense_max_packets[0].Get<std::vector<float>>();
@@ -430,13 +457,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksImageWithPostStreamFloatList) {
   mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
                             input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("IMAGE").packets;
+      runner_->Outputs().Tag(kImageTag).packets;
   ASSERT_EQ(num_images, output_packets.size());
 
   for (int i = 0; i < num_images; ++i) {
@@ -463,13 +490,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) {
   mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
                             input_sequence.get());
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(input_sequence.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& fdense_max_packets =
-      runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets;
+      runner_->Outputs().Tag(kFloatFeatureFdenseMaxTag).packets;
   ASSERT_EQ(fdense_max_packets.size(), 1);
   const auto& fdense_max_vector =
       fdense_max_packets[0].Get<std::vector<float>>();
@@ -481,17 +508,17 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) {
 TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromPacket) {
   SetUpCalculator({}, {"DATA_PATH:data_path"}, {"DATASET_ROOT:root"});
 
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
 
   std::string root = "test_root";
-  runner_->MutableSidePackets()->Tag("DATASET_ROOT") = PointToForeign(&root);
+  runner_->MutableSidePackets()->Tag(kDatasetRootTag) = PointToForeign(&root);
   MP_ASSERT_OK(runner_->Run());
 
   MP_ASSERT_OK(runner_->OutputSidePackets()
-                   .Tag("DATA_PATH")
+                   .Tag(kDataPathTag)
                    .ValidateAsType<std::string>());
-  ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
+  ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
             root + "/" + data_path_);
 }
 
@@ -501,28 +528,28 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromOptions) {
   options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
       ->set_dataset_root_directory(root);
   SetUpCalculator({}, {"DATA_PATH:data_path"}, {}, &options);
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
 
   MP_ASSERT_OK(runner_->Run());
 
   MP_ASSERT_OK(runner_->OutputSidePackets()
-                   .Tag("DATA_PATH")
+                   .Tag(kDataPathTag)
                    .ValidateAsType<std::string>());
-  ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
+  ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
             root + "/" + data_path_);
 }
 
 TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromExample) {
   SetUpCalculator({}, {"DATA_PATH:data_path"});
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
   MP_ASSERT_OK(runner_->Run());
 
   MP_ASSERT_OK(runner_->OutputSidePackets()
-                   .Tag("DATA_PATH")
+                   .Tag(kDataPathTag)
                    .ValidateAsType<std::string>());
-  ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
+  ASSERT_EQ(runner_->OutputSidePackets().Tag(kDataPathTag).Get<std::string>(),
             data_path_);
 }
 
@@ -534,20 +561,20 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptions) {
       ->set_padding_after_label(2);
   SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
                   &options);
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
   MP_ASSERT_OK(runner_->Run());
 
   MP_EXPECT_OK(runner_->OutputSidePackets()
-                   .Tag("AUDIO_DECODER_OPTIONS")
+                   .Tag(kAudioDecoderOptionsTag)
                    .ValidateAsType<AudioDecoderOptions>());
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Tag(kAudioDecoderOptionsTag)
                   .Get<AudioDecoderOptions>()
                   .start_time(),
               2.0, 1e-5);
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Tag(kAudioDecoderOptionsTag)
                   .Get<AudioDecoderOptions>()
                   .end_time(),
               7.0, 1e-5);
@@ -563,20 +590,20 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptionsOverride) {
       ->set_force_decoding_from_start_of_media(true);
   SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
                   &options);
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
   MP_ASSERT_OK(runner_->Run());
 
   MP_EXPECT_OK(runner_->OutputSidePackets()
-                   .Tag("AUDIO_DECODER_OPTIONS")
+                   .Tag(kAudioDecoderOptionsTag)
                    .ValidateAsType<AudioDecoderOptions>());
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Tag(kAudioDecoderOptionsTag)
                   .Get<AudioDecoderOptions>()
                   .start_time(),
               0.0, 1e-5);
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Tag(kAudioDecoderOptionsTag)
                   .Get<AudioDecoderOptions>()
                   .end_time(),
               7.0, 1e-5);
@@ -594,27 +621,27 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
       ->mutable_base_packet_resampler_options()
       ->set_frame_rate(1.0);
   SetUpCalculator({}, {"RESAMPLER_OPTIONS:resampler_options"}, {}, &options);
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
   MP_ASSERT_OK(runner_->Run());
 
   MP_EXPECT_OK(runner_->OutputSidePackets()
-                   .Tag("RESAMPLER_OPTIONS")
+                   .Tag(kResamplerOptionsTag)
                    .ValidateAsType<CalculatorOptions>());
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("RESAMPLER_OPTIONS")
+                  .Tag(kResamplerOptionsTag)
                   .Get<CalculatorOptions>()
                   .GetExtension(PacketResamplerCalculatorOptions::ext)
                   .start_time(),
               2000000, 1);
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("RESAMPLER_OPTIONS")
+                  .Tag(kResamplerOptionsTag)
                   .Get<CalculatorOptions>()
                   .GetExtension(PacketResamplerCalculatorOptions::ext)
                   .end_time(),
               7000000, 1);
   EXPECT_NEAR(runner_->OutputSidePackets()
-                  .Tag("RESAMPLER_OPTIONS")
+                  .Tag(kResamplerOptionsTag)
                   .Get<CalculatorOptions>()
                   .GetExtension(PacketResamplerCalculatorOptions::ext)
                   .frame_rate(),
@@ -623,13 +650,13 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
 
 TEST_F(UnpackMediaSequenceCalculatorTest, GetFrameRateFromExample) {
   SetUpCalculator({}, {"IMAGE_FRAME_RATE:frame_rate"});
-  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
       Adopt(sequence_.release());
   MP_ASSERT_OK(runner_->Run());
   MP_EXPECT_OK(runner_->OutputSidePackets()
-                   .Tag("IMAGE_FRAME_RATE")
+                   .Tag(kImageFrameRateTag)
                    .ValidateAsType<double>());
-  EXPECT_EQ(runner_->OutputSidePackets().Tag("IMAGE_FRAME_RATE").Get<double>(),
+  EXPECT_EQ(runner_->OutputSidePackets().Tag(kImageFrameRateTag).Get<double>(),
             image_frame_rate_);
 }
 

mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc

@@ -26,6 +26,10 @@ namespace {
 
 namespace tf = ::tensorflow;
 
+constexpr char kSingleIntTag[] = "SINGLE_INT";
+constexpr char kTensorOutTag[] = "TENSOR_OUT";
+constexpr char kVectorIntTag[] = "VECTOR_INT";
+
 class VectorIntToTensorCalculatorTest : public ::testing::Test {
  protected:
   void SetUpRunner(
@@ -61,13 +65,13 @@ class VectorIntToTensorCalculatorTest : public ::testing::Test {
 
     const int64 time = 1234;
     runner_->MutableInputs()
-        ->Tag("VECTOR_INT")
+        ->Tag(kVectorIntTag)
         .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
 
     EXPECT_TRUE(runner_->Run().ok());
 
     const std::vector<Packet>& output_packets =
-        runner_->Outputs().Tag("TENSOR_OUT").packets;
+        runner_->Outputs().Tag(kTensorOutTag).packets;
     EXPECT_EQ(1, output_packets.size());
     EXPECT_EQ(time, output_packets[0].Timestamp().Value());
     const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
@@ -95,13 +99,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestSingleValue) {
               tensorflow::DT_INT32, false, true);
   const int64 time = 1234;
   runner_->MutableInputs()
-      ->Tag("SINGLE_INT")
+      ->Tag(kSingleIntTag)
       .packets.push_back(MakePacket<int>(1).At(Timestamp(time)));
 
   EXPECT_TRUE(runner_->Run().ok());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("TENSOR_OUT").packets;
+      runner_->Outputs().Tag(kTensorOutTag).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
@@ -121,13 +125,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TesOneDim) {
   }
   const int64 time = 1234;
   runner_->MutableInputs()
-      ->Tag("VECTOR_INT")
+      ->Tag(kVectorIntTag)
       .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
 
   EXPECT_TRUE(runner_->Run().ok());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("TENSOR_OUT").packets;
+      runner_->Outputs().Tag(kTensorOutTag).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
@@ -152,13 +156,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestInt64) {
               tensorflow::DT_INT64, false, true);
   const int64 time = 1234;
   runner_->MutableInputs()
-      ->Tag("SINGLE_INT")
+      ->Tag(kSingleIntTag)
       .packets.push_back(MakePacket<int>(1LL << 31).At(Timestamp(time)));
 
   EXPECT_TRUE(runner_->Run().ok());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("TENSOR_OUT").packets;
+      runner_->Outputs().Tag(kTensorOutTag).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
@@ -179,13 +183,13 @@ TEST_F(VectorIntToTensorCalculatorTest, TestUint8) {
   }
   const int64 time = 1234;
   runner_->MutableInputs()
-      ->Tag("VECTOR_INT")
+      ->Tag(kVectorIntTag)
       .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
 
   EXPECT_TRUE(runner_->Run().ok());
 
   const std::vector<Packet>& output_packets =
-      runner_->Outputs().Tag("TENSOR_OUT").packets;
+      runner_->Outputs().Tag(kTensorOutTag).packets;
   EXPECT_EQ(1, output_packets.size());
   EXPECT_EQ(time, output_packets[0].Timestamp().Value());
   const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();

mediapipe/calculators/tflite/BUILD

@@ -162,6 +162,27 @@ selects.config_setting_group(
     ],
 )
 
+config_setting(
+    name = "edge_tpu_usb",
+    define_values = {
+        "MEDIAPIPE_EDGE_TPU": "usb",
+    },
+)
+
+config_setting(
+    name = "edge_tpu_pci",
+    define_values = {
+        "MEDIAPIPE_EDGE_TPU": "pci",
+    },
+)
+
+config_setting(
+    name = "edge_tpu_all",
+    define_values = {
+        "MEDIAPIPE_EDGE_TPU": "all",
+    },
+)
+
 cc_library(
     name = "tflite_inference_calculator",
     srcs = ["tflite_inference_calculator.cc"],
@@ -172,6 +193,12 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    defines = select({
+        "//conditions:default": [],
+        ":edge_tpu_usb": ["MEDIAPIPE_EDGE_TPU=usb"],
+        ":edge_tpu_pci": ["MEDIAPIPE_EDGE_TPU=pci"],
+        ":edge_tpu_all": ["MEDIAPIPE_EDGE_TPU=all"],
+    }),
     linkopts = select({
         "//mediapipe:ios": [
             "-framework CoreVideo",
@@ -223,6 +250,20 @@ cc_library(
         "//conditions:default": [
             "//mediapipe/util:cpu_util",
         ],
+    }) + select({
+        "//conditions:default": [],
+        ":edge_tpu_usb": [
+            "@libedgetpu//tflite/public:edgetpu",
+            "@libedgetpu//tflite/public:oss_edgetpu_direct_usb",
+        ],
+        ":edge_tpu_pci": [
+            "@libedgetpu//tflite/public:edgetpu",
+            "@libedgetpu//tflite/public:oss_edgetpu_direct_pci",
+        ],
+        ":edge_tpu_all": [
+            "@libedgetpu//tflite/public:edgetpu",
+            "@libedgetpu//tflite/public:oss_edgetpu_direct_all",
+        ],
     }),
     alwayslink = 1,
 )

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -85,7 +85,22 @@ constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 #if defined(MEDIAPIPE_EDGE_TPU)
-#include "edgetpu.h"
+#include "tflite/public/edgetpu.h"
+
+// Checkes whether model contains Edge TPU custom op or not.
+bool ContainsEdgeTpuCustomOp(const tflite::FlatBufferModel& model) {
+  const auto* opcodes = model.GetModel()->operator_codes();
+  for (const auto* subgraph : *model.GetModel()->subgraphs()) {
+    for (const auto* op : *subgraph->operators()) {
+      const auto* opcode = opcodes->Get(op->opcode_index());
+      if (opcode->custom_code() &&
+          opcode->custom_code()->str() == edgetpu::kCustomOp) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
 
 // Creates and returns an Edge TPU interpreter to run the given edgetpu model.
 std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
@@ -94,14 +109,9 @@ std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
     edgetpu::EdgeTpuContext* edgetpu_context) {
   resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
   std::unique_ptr<tflite::Interpreter> interpreter;
-  if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) {
+  CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter),
-    std::cerr << "Failed to build edge TPU interpreter." << std::endl;
+           kTfLiteOk);
-  }
   interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
-  interpreter->SetNumThreads(1);
-  if (interpreter->AllocateTensors() != kTfLiteOk) {
-    std::cerr << "Failed to allocate edge TPU tensors." << std::endl;
-  }
   return interpreter;
 }
 #endif  // MEDIAPIPE_EDGE_TPU
@@ -279,8 +289,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
 #endif  // MEDIAPIPE_TFLITE_GL_INFERENCE
 
 #if defined(MEDIAPIPE_EDGE_TPU)
-  std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_ =
+  std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_;
-      edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
 #endif
 
   bool gpu_inference_ = false;
@@ -292,6 +301,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   bool allow_precision_loss_ = false;
   mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api
       tflite_gpu_runner_api_;
+  mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
+      tflite_gpu_runner_usage_;
 
   bool use_kernel_caching_ = false;
   std::string cached_kernel_filename_;
@@ -301,6 +312,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 // Calculator Core Section
 
 namespace {
+
+constexpr char kCustomOpResolverTag[] = "CUSTOM_OP_RESOLVER";
+constexpr char kModelTag[] = "MODEL";
+
 template <class CC>
 bool ShouldUseGpu(CC* cc) {
 #if MEDIAPIPE_TFLITE_GPU_SUPPORTED
@@ -325,7 +340,7 @@ absl::Status TfLiteInferenceCalculator::GetContract(CalculatorContract* cc) {
   const auto& options =
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
   RET_CHECK(!options.model_path().empty() ^
-            cc->InputSidePackets().HasTag("MODEL"))
+            cc->InputSidePackets().HasTag(kModelTag))
       << "Either model as side packet or model path in options is required.";
 
   if (cc->Inputs().HasTag(kTensorsTag))
@@ -338,13 +353,13 @@ absl::Status TfLiteInferenceCalculator::GetContract(CalculatorContract* cc) {
   if (cc->Outputs().HasTag(kTensorsGpuTag))
     cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
 
-  if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
+  if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
     cc->InputSidePackets()
-        .Tag("CUSTOM_OP_RESOLVER")
+        .Tag(kCustomOpResolverTag)
         .Set<tflite::ops::builtin::BuiltinOpResolver>();
   }
-  if (cc->InputSidePackets().HasTag("MODEL")) {
+  if (cc->InputSidePackets().HasTag(kModelTag)) {
-    cc->InputSidePackets().Tag("MODEL").Set<TfLiteModelPtr>();
+    cc->InputSidePackets().Tag(kModelTag).Set<TfLiteModelPtr>();
   }
 
   if (ShouldUseGpu(cc)) {
@@ -377,6 +392,7 @@ absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) {
                           options.delegate().gpu().use_advanced_gpu_api();
   allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
   tflite_gpu_runner_api_ = options.delegate().gpu().api();
+  tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
 
   use_kernel_caching_ = use_advanced_gpu_api_ &&
                         options.delegate().gpu().has_cached_kernel_path();
@@ -483,8 +499,8 @@ absl::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
   MP_RETURN_IF_ERROR(WriteKernelsToFile());
 
   return RunInContextIfNeeded([this]() -> absl::Status {
+    interpreter_ = nullptr;
     if (delegate_) {
-      interpreter_ = nullptr;
       delegate_ = nullptr;
 #if MEDIAPIPE_TFLITE_GPU_SUPPORTED
       if (gpu_inference_) {
@@ -498,7 +514,7 @@ absl::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
 #endif  // MEDIAPIPE_TFLITE_GPU_SUPPORTED
     }
 #if defined(MEDIAPIPE_EDGE_TPU)
-    edgetpu_context_.reset();
+    edgetpu_context_ = nullptr;
 #endif
     return absl::OkStatus();
   });
@@ -720,9 +736,9 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
   auto op_resolver_ptr =
       static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
           &default_op_resolver);
-  if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
+  if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
     op_resolver_ptr = &(cc->InputSidePackets()
-                            .Tag("CUSTOM_OP_RESOLVER")
+                            .Tag(kCustomOpResolverTag)
                             .Get<tflite::ops::builtin::BuiltinOpResolver>());
   }
 
@@ -733,7 +749,23 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
                           : tflite::gpu::InferencePriority::MAX_PRECISION;
   options.priority2 = tflite::gpu::InferencePriority::AUTO;
   options.priority3 = tflite::gpu::InferencePriority::AUTO;
-  options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+  switch (tflite_gpu_runner_usage_) {
+    case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
+        FAST_SINGLE_ANSWER: {
+      options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
+      break;
+    }
+    case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
+        SUSTAINED_SPEED: {
+      options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+      break;
+    }
+    case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::
+        UNSPECIFIED: {
+      return absl::InternalError("inference usage need to be specified.");
+    }
+  }
+
   tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
   switch (tflite_gpu_runner_api_) {
     case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::OPENGL: {
@@ -806,21 +838,26 @@ absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) {
 
   tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates
       default_op_resolver;
-  auto op_resolver_ptr =
-      static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
-          &default_op_resolver);
-
-  if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
-    op_resolver_ptr = &(cc->InputSidePackets()
-                            .Tag("CUSTOM_OP_RESOLVER")
-                            .Get<tflite::ops::builtin::BuiltinOpResolver>());
-  }
-
 #if defined(MEDIAPIPE_EDGE_TPU)
-  interpreter_ =
+  if (ContainsEdgeTpuCustomOp(model)) {
-      BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get());
+    edgetpu_context_ = edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
-#else
+    interpreter_ = BuildEdgeTpuInterpreter(model, &default_op_resolver,
-  tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
+                                           edgetpu_context_.get());
+  } else {
+#endif  // MEDIAPIPE_EDGE_TPU
+    auto op_resolver_ptr =
+        static_cast<const tflite::ops::builtin::BuiltinOpResolver*>(
+            &default_op_resolver);
+
+    if (cc->InputSidePackets().HasTag(kCustomOpResolverTag)) {
+      op_resolver_ptr = &(cc->InputSidePackets()
+                              .Tag(kCustomOpResolverTag)
+                              .Get<tflite::ops::builtin::BuiltinOpResolver>());
+    }
+
+    tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_);
+#if defined(MEDIAPIPE_EDGE_TPU)
+  }
 #endif  // MEDIAPIPE_EDGE_TPU
 
   RET_CHECK(interpreter_);
@@ -853,8 +890,8 @@ absl::StatusOr<Packet> TfLiteInferenceCalculator::GetModelAsPacket(
   if (!options.model_path().empty()) {
     return TfLiteModelLoader::LoadFromPath(options.model_path());
   }
-  if (cc.InputSidePackets().HasTag("MODEL")) {
+  if (cc.InputSidePackets().HasTag(kModelTag)) {
-    return cc.InputSidePackets().Tag("MODEL");
+    return cc.InputSidePackets().Tag(kModelTag);
   }
   return absl::Status(absl::StatusCode::kNotFound,
                       "Must specify TFLite model as path or loaded model.");
@@ -878,11 +915,15 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) {
       // Attempt to use NNAPI.
       // If not supported, the default CPU delegate will be created and used.
       interpreter_->SetAllowFp16PrecisionForFp32(1);
-      delegate_ =
+      tflite::StatefulNnApiDelegate::Options options;
-          TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
+      const auto& nnapi = calculator_opts.delegate().nnapi();
-            // No need to free according to tflite::NnApiDelegate()
+      // Set up cache_dir and model_token for NNAPI compilation cache.
-            // documentation.
+      if (nnapi.has_cache_dir() && nnapi.has_model_token()) {
-          });
+        options.cache_dir = nnapi.cache_dir().c_str();
+        options.model_token = nnapi.model_token().c_str();
+      }
+      delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
+                                    [](TfLiteDelegate*) {});
       RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
                    kTfLiteOk);
       return absl::OkStatus();
@@ -906,6 +947,8 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) {
                    kTfLiteOk);
       return absl::OkStatus();
     }
+#else
+    (void)use_xnnpack;
 #endif  // !EDGETPU
 
     // Return and use default tflite infernece (on CPU). No need for GPU

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -67,9 +67,31 @@ message TfLiteInferenceCalculatorOptions {
       // Only available for OpenCL delegate on Android.
       // Kernel caching will only be enabled if this path is set.
       optional string cached_kernel_path = 2;
+
+      // Encapsulated compilation/runtime tradeoffs.
+      enum InferenceUsage {
+        UNSPECIFIED = 0;
+
+        // InferenceRunner will be used only once. Therefore, it is important to
+        // minimize bootstrap time as well.
+        FAST_SINGLE_ANSWER = 1;
+
+        // Prefer maximizing the throughput. Same inference runner will be used
+        // repeatedly on different inputs.
+        SUSTAINED_SPEED = 2;
+      }
+      optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED];
     }
     // Android only.
-    message Nnapi {}
+    message Nnapi {
+      // Directory to store compilation cache. If unspecified, NNAPI will not
+      // try caching the compilation.
+      optional string cache_dir = 1;
+      // Unique token identifying the model. It is the caller's responsibility
+      // to ensure there is no clash of the tokens. If unspecified, NNAPI will
+      // not try caching the compilation.
+      optional string model_token = 2;
+    }
     message Xnnpack {
       // Number of threads for XNNPACK delegate. (By default, calculator tries
       // to choose optimal number of threads depending on the device.)

mediapipe/calculators/tflite/tflite_tensors_to_floats_calculator.cc

@@ -18,6 +18,10 @@
 
 namespace mediapipe {
 
+constexpr char kFloatsTag[] = "FLOATS";
+constexpr char kFloatTag[] = "FLOAT";
+constexpr char kTensorsTag[] = "TENSORS";
+
 // A calculator for converting TFLite tensors to to a float or a float vector.
 //
 // Input:
@@ -48,15 +52,16 @@ REGISTER_CALCULATOR(TfLiteTensorsToFloatsCalculator);
 
 absl::Status TfLiteTensorsToFloatsCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("TENSORS"));
+  RET_CHECK(cc->Inputs().HasTag(kTensorsTag));
-  RET_CHECK(cc->Outputs().HasTag("FLOATS") || cc->Outputs().HasTag("FLOAT"));
+  RET_CHECK(cc->Outputs().HasTag(kFloatsTag) ||
+            cc->Outputs().HasTag(kFloatTag));
 
-  cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
-  if (cc->Outputs().HasTag("FLOATS")) {
+  if (cc->Outputs().HasTag(kFloatsTag)) {
-    cc->Outputs().Tag("FLOATS").Set<std::vector<float>>();
+    cc->Outputs().Tag(kFloatsTag).Set<std::vector<float>>();
   }
-  if (cc->Outputs().HasTag("FLOAT")) {
+  if (cc->Outputs().HasTag(kFloatTag)) {
-    cc->Outputs().Tag("FLOAT").Set<float>();
+    cc->Outputs().Tag(kFloatTag).Set<float>();
   }
 
   return absl::OkStatus();
@@ -69,10 +74,10 @@ absl::Status TfLiteTensorsToFloatsCalculator::Open(CalculatorContext* cc) {
 }
 
 absl::Status TfLiteTensorsToFloatsCalculator::Process(CalculatorContext* cc) {
-  RET_CHECK(!cc->Inputs().Tag("TENSORS").IsEmpty());
+  RET_CHECK(!cc->Inputs().Tag(kTensorsTag).IsEmpty());
 
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
   // TODO: Add option to specify which tensor to take from.
   const TfLiteTensor* raw_tensor = &input_tensors[0];
   const float* raw_floats = raw_tensor->data.f;
@@ -82,18 +87,19 @@ absl::Status TfLiteTensorsToFloatsCalculator::Process(CalculatorContext* cc) {
     num_values *= raw_tensor->dims->data[i];
   }
 
-  if (cc->Outputs().HasTag("FLOAT")) {
+  if (cc->Outputs().HasTag(kFloatTag)) {
     // TODO: Could add an index in the option to specifiy returning one
     // value of a float array.
     RET_CHECK_EQ(num_values, 1);
-    cc->Outputs().Tag("FLOAT").AddPacket(
+    cc->Outputs().Tag(kFloatTag).AddPacket(
         MakePacket<float>(raw_floats[0]).At(cc->InputTimestamp()));
   }
-  if (cc->Outputs().HasTag("FLOATS")) {
+  if (cc->Outputs().HasTag(kFloatsTag)) {
     auto output_floats = absl::make_unique<std::vector<float>>(
         raw_floats, raw_floats + num_values);
-    cc->Outputs().Tag("FLOATS").Add(output_floats.release(),
+    cc->Outputs()
-                                    cc->InputTimestamp());
+        .Tag(kFloatsTag)
+        .Add(output_floats.release(), cc->InputTimestamp());
   }
 
   return absl::OkStatus();