Project import generated by Copybara.

GitOrigin-RevId: 6e5aa035cd1f6a9333962df5d3ab97a05bd5744e

.bazelrc

@@ -32,6 +32,9 @@ build:macos --copt=-w
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
 
+# Compile ObjC++ files with C++17
+build --per_file_copt=.*\.mm\$@-std=c++17
+
 # Allow debugging with XCODE
 build --apple_generate_dsym
 
@@ -58,6 +61,7 @@ build:android_arm64 --fat_apk_cpu=arm64-v8a
 
 # iOS configs.
 build:ios --apple_platform_type=ios
+build:ios --copt=-fno-aligned-allocation
 
 build:ios_i386 --config=ios
 build:ios_i386 --cpu=ios_i386
@@ -87,6 +91,10 @@ build:darwin_x86_64 --apple_platform_type=macos
 build:darwin_x86_64 --macos_minimum_os=10.12
 build:darwin_x86_64 --cpu=darwin_x86_64
 
+build:darwin_arm64 --apple_platform_type=macos
+build:darwin_arm64 --macos_minimum_os=10.16
+build:darwin_arm64 --cpu=darwin_arm64
+
 # This bazelrc file is meant to be written by a setup script.
 try-import %workspace%/.configure.bazelrc
 

.bazelversion

@@ -1 +1 @@
-3.7.2
+5.2.0

.github/ISSUE_TEMPLATE/50-other-issues.md

@@ -10,5 +10,3 @@ For questions on how to work with MediaPipe, or support for problems that are no
 
 If you are reporting a vulnerability, please use the [dedicated reporting process](https://github.com/google/mediapipe/security).
 
-For high-level discussions about MediaPipe, please post to discuss@mediapipe.org, for questions about the development or internal workings of MediaPipe, or if you would like to know how to contribute to MediaPipe, please post to developers@mediapipe.org.
-

.github/bot_config.yml

@@ -15,4 +15,4 @@
 
 # A list of assignees
 assignees:
-   - sgowroji
+   - sureshdagooglecom

CONTRIBUTING.md

@@ -5,7 +5,7 @@
 * Bug fixes
 * Documentation fixes
 
-For new feature additions (e.g., new graphs and calculators), we are currently not planning to accept new feature pull requests into the MediaPipe repository. Instead, we like to get contributors to create their own repositories of the new feature and list it at [Awesome MediaPipe](https://mediapipe.org). This will allow contributors to more quickly get their code out to the community.
+For new feature additions (e.g., new graphs and calculators), we are currently not planning to accept new feature pull requests into the MediaPipe repository. Instead, we like to get contributors to create their own repositories of the new feature and list it at [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe). This will allow contributors to more quickly get their code out to the community.
 
 Before sending your pull requests, make sure you followed this list.
 

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 MAINTAINER <[email protected]>
 
@@ -42,6 +42,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         software-properties-common && \
     add-apt-repository -y ppa:openjdk-r/ppa && \
     apt-get update && apt-get install -y openjdk-8-jdk && \
+    apt-get install -y mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev && \
+    apt-get install -y mesa-utils && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -50,13 +52,13 @@ RUN pip3 install --upgrade setuptools
 RUN pip3 install wheel
 RUN pip3 install future
 RUN pip3 install six==1.14.0
-RUN pip3 install tensorflow==1.14.0
+RUN pip3 install tensorflow==2.2.0
 RUN pip3 install tf_slim
 
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.7.2
+ARG BAZEL_VERSION=5.2.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

README.md

@@ -136,8 +136,8 @@ run code search using
 
 ## Community
 
-*   [Awesome MediaPipe](https://mediapipe.org) - A curated list of awesome
-    MediaPipe related frameworks, libraries and software
+*   [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
+    curated list of awesome MediaPipe related frameworks, libraries and software
 *   [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
 *   [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
     community discussion around MediaPipe

WORKSPACE

@@ -16,11 +16,11 @@ bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
 versions.check(minimum_bazel_version = "3.7.2")
 
-# ABSL cpp library lts_2020_09_23
+# ABSL cpp library lts_2021_03_24, patch 2.
 http_archive(
     name = "com_google_absl",
     urls = [
-        "https://github.com/abseil/abseil-cpp/archive/20200923.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz",
     ],
     # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
     patches = [
@@ -29,14 +29,15 @@ http_archive(
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "abseil-cpp-20200923",
-    sha256 = "b3744a4f7a249d5eaf2309daad597631ce77ea62e0fc6abffbab4b4c3dc0fc08"
+    strip_prefix = "abseil-cpp-20210324.2",
+    sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f"
 )
 
 http_archive(
     name = "rules_cc",
-    strip_prefix = "rules_cc-main",
-    urls = ["https://github.com/bazelbuild/rules_cc/archive/main.zip"],
+    strip_prefix = "rules_cc-2f8c04c04462ab83c545ab14c0da68c3b4c96191",
+# The commit can be updated if the build passes. Last updated 6/23/22.
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/2f8c04c04462ab83c545ab14c0da68c3b4c96191.zip"],
 )
 
 http_archive(
@@ -61,11 +62,12 @@ http_archive(
     sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049",
 )
 
-# Google Benchmark library.
+# Google Benchmark library v1.6.1 released on 2022-01-10.
 http_archive(
     name = "com_google_benchmark",
-    urls = ["https://github.com/google/benchmark/archive/main.zip"],
-    strip_prefix = "benchmark-main",
+    urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.6.1.tar.gz"],
+    strip_prefix = "benchmark-1.6.1",
+    sha256 = "6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4",
     build_file = "@//third_party:benchmark.BUILD",
 )
 
@@ -122,16 +124,16 @@ http_archive(
 # ...but the Java download is currently broken, so we use the "source" download.
 http_archive(
     name = "com_google_protobuf_javalite",
-    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
+    sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
+    strip_prefix = "protobuf-3.19.1",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
 )
 
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
+    sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
+    strip_prefix = "protobuf-3.19.1",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
     patches = [
         "@//third_party:com_google_protobuf_fixes.diff"
     ],
@@ -154,28 +156,29 @@ http_archive(
     sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91",
 )
 
+# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
 http_archive(
     name = "pybind11",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
-        "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
+        "https://github.com/pybind/pybind11/archive/70a58c577eaf067748c2ec31bfd0b0a614cffba6.zip",
     ],
-    sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d",
-    strip_prefix = "pybind11-2.4.3",
+    sha256 = "b971842fab1b5b8f3815a2302331782b7d137fef0e06502422bc4bc360f4956c",
+    strip_prefix = "pybind11-70a58c577eaf067748c2ec31bfd0b0a614cffba6",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
+# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
 http_archive(
     name = "ceres_solver",
-    url = "https://github.com/ceres-solver/ceres-solver/archive/2.0.0.zip",
+    url = "https://github.com/ceres-solver/ceres-solver/archive/123fba61cf2611a3c8bddc9d91416db26b10b558.zip",
     patches = [
         "@//third_party:ceres_solver_compatibility_fixes.diff"
     ],
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "ceres-solver-2.0.0",
-    sha256 = "db12d37b4cebb26353ae5b7746c7985e00877baa8e7b12dc4d3a1512252fff3b"
+    strip_prefix = "ceres-solver-123fba61cf2611a3c8bddc9d91416db26b10b558",
+    sha256 = "8b7b16ceb363420e0fd499576daf73fa338adb0b1449f58bea7862766baa1ac7"
 )
 
 http_archive(
@@ -200,7 +203,10 @@ new_local_repository(
 new_local_repository(
     name = "macos_opencv",
     build_file = "@//third_party:opencv_macos.BUILD",
-    path = "/usr/local/opt/opencv@3",
+    # For local MacOS builds, the path should point to an opencv@3 installation.
+    # If you edit the path here, you will also need to update the corresponding
+    # prefix in "opencv_macos.BUILD".
+    path = "/usr/local",
 )
 
 new_local_repository(
@@ -249,21 +255,12 @@ http_archive(
     ],
 )
 
-# You may run setup_android.sh to install Android SDK and NDK.
-android_ndk_repository(
-    name = "androidndk",
-)
-
-android_sdk_repository(
-    name = "androidsdk",
-)
-
 # iOS basic build deps.
 
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42",
-    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz",
+    sha256 = "77e8bf6fda706f420a55874ae6ee4df0c9d95da6c7838228b26910fc82eea5a2",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.32.0/rules_apple.0.32.0.tar.gz",
     patches = [
         # Bypass checking ios unit test runner when building MP ios applications.
         "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
@@ -289,10 +286,9 @@ swift_rules_dependencies()
 
 http_archive(
     name = "build_bazel_apple_support",
-    sha256 = "122ebf7fe7d1c8e938af6aeaee0efe788a3a2449ece5a8d6a428cb18d6f88033",
+    sha256 = "741366f79d900c11e11d8efd6cc6c66a31bfb2451178b58e0b5edc6f1db17b35",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
-        "https://github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
+        "https://github.com/bazelbuild/apple_support/releases/download/0.10.0/apple_support.0.10.0.tar.gz"
     ],
 )
 
@@ -333,6 +329,7 @@ maven_install(
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.3.1",
         "androidx.activity:activity:1.2.2",
+        "androidx.exifinterface:exifinterface:1.3.3",
         "androidx.fragment:fragment:1.3.4",
         "androidx.annotation:annotation:aar:1.1.0",
         "androidx.appcompat:appcompat:aar:1.1.0-rc01",
@@ -349,8 +346,11 @@ maven_install(
         "com.google.auto.value:auto-value:1.8.1",
         "com.google.auto.value:auto-value-annotations:1.8.1",
         "com.google.code.findbugs:jsr305:latest.release",
-        "com.google.flogger:flogger-system-backend:latest.release",
-        "com.google.flogger:flogger:latest.release",
+        "com.google.android.datatransport:transport-api:3.0.0",
+        "com.google.android.datatransport:transport-backend-cct:3.1.0",
+        "com.google.android.datatransport:transport-runtime:3.1.0",
+        "com.google.flogger:flogger-system-backend:0.6",
+        "com.google.flogger:flogger:0.6",
         "com.google.guava:guava:27.0.1-android",
         "com.google.guava:listenablefuture:1.0",
         "junit:junit:4.12",
@@ -378,9 +378,9 @@ http_archive(
 )
 
 # Tensorflow repo should always go after the other external dependencies.
-# 2021-07-29
-_TENSORFLOW_GIT_COMMIT = "52a2905cbc21034766c08041933053178c5d10e3"
-_TENSORFLOW_SHA256 = "06d4691bcdb700f3275fa0971a1585221c2b9f3dffe867963be565a6643d7f56"
+# 2022-02-15
+_TENSORFLOW_GIT_COMMIT = "a3419acc751dfc19caf4d34a1594e1f76810ec58"
+_TENSORFLOW_SHA256 = "b95b2a83632d4055742ae1a2dcc96b45da6c12a339462dbc76c8bca505308e3a"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -388,7 +388,8 @@ http_archive(
     ],
     patches = [
         "@//third_party:org_tensorflow_compatibility_fixes.diff",
-        "@//third_party:org_tensorflow_objc_cxx17.diff",
+        # Diff is generated with a script, don't update it manually.
+        "@//third_party:org_tensorflow_custom_ops.diff",
     ],
     patch_args = [
         "-p1",

build_android_examples.sh

@@ -109,7 +109,7 @@ for app in ${apps}; do
           if [[ ${category} != "shoe" ]]; then
             bazel_flags_extended+=(--define ${category}=true)
           fi
-          bazel "${bazel_flags_extended[@]}"
+          bazelisk "${bazel_flags_extended[@]}"
           cp -f "${bin}" "${apk}"
         fi
         apks+=(${apk})
@@ -120,7 +120,7 @@ for app in ${apps}; do
         if [[ ${app_name} == "templatematchingcpu" ]]; then
           switch_to_opencv_4
         fi
-        bazel "${bazel_flags[@]}"
+        bazelisk "${bazel_flags[@]}"
         cp -f "${bin}" "${apk}"
         if [[ ${app_name} == "templatematchingcpu" ]]; then
           switch_to_opencv_3

build_desktop_examples.sh

@@ -83,7 +83,7 @@ for app in ${apps}; do
       bazel_flags=("${default_bazel_flags[@]}")
       bazel_flags+=(${target})
 
-      bazel "${bazel_flags[@]}"
+      bazelisk "${bazel_flags[@]}"
       cp -f "${bin_dir}/${app}/"*"_cpu" "${out_dir}"
     fi
     if [[ $build_only == false ]]; then

build_ios_examples.sh

@@ -71,7 +71,7 @@ for app in ${apps}; do
       bazel_flags+=(--linkopt=-s)
     fi
 
-    bazel "${bazel_flags[@]}"
+    bazelisk "${bazel_flags[@]}"
     cp -f "${bin_dir}/${app}/"*".ipa" "${out_dir}"
   fi
 done

docs/framework_concepts/synchronization.md

@@ -169,7 +169,7 @@ behavior depending on resource constraints.
 
 [`CalculatorBase`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h
 [`DefaultInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/default_input_stream_handler.h
-[`SyncSetInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/sync_set_input_stream_handler.h
-[`ImmediateInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/immediate_input_stream_handler.h
+[`SyncSetInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc
+[`ImmediateInputStreamHandler`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc
 [`CalculatorGraphConfig::max_queue_size`]: https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto
 [`FlowLimiterCalculator`]: https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/flow_limiter_calculator.cc

docs/getting_started/android.md

@@ -29,7 +29,7 @@ APIs (currently in alpha) that are now available in
 
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
-*   Setup Android SDK release 28.0.3 and above.
+*   Setup Android SDK release 30.0.0 and above.
 *   Setup Android NDK version between 18 and 21.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
@@ -47,6 +47,15 @@ export ANDROID_HOME=<path to the Android SDK>
 export ANDROID_NDK_HOME=<path to the Android NDK>
 ```
 
+and add android_ndk_repository() and android_sdk_repository() rules into the
+[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE) file as
+the following:
+
+```bash
+$ echo "android_sdk_repository(name = \"androidsdk\")" >> WORKSPACE
+$ echo "android_ndk_repository(name = \"androidndk\", api_level=21)" >> WORKSPACE
+```
+
 In order to use MediaPipe on earlier Android versions, MediaPipe needs to switch
 to a lower Android API level. You can achieve this by specifying `api_level =
 $YOUR_INTENDED_API_LEVEL` in android_ndk_repository() and/or

docs/getting_started/android_archive_library.md

@@ -48,6 +48,16 @@ each project.
     bazel build -c opt --strip=ALWAYS \
         --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
         --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        --legacy_whole_archive=0 \
+        --features=-legacy_whole_archive \
+        --copt=-fvisibility=hidden \
+        --copt=-ffunction-sections \
+        --copt=-fdata-sections \
+        --copt=-fstack-protector \
+        --copt=-Oz \
+        --copt=-fomit-frame-pointer \
+        --copt=-DABSL_MIN_LOG_LEVEL=2 \
+        --linkopt=-Wl,--gc-sections,--strip-all \
         //path/to/the/aar/build/file:aar_name.aar
     ```
 
@@ -57,6 +67,16 @@ each project.
     bazel build -c opt --strip=ALWAYS \
         --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
         --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        --legacy_whole_archive=0 \
+        --features=-legacy_whole_archive \
+        --copt=-fvisibility=hidden \
+        --copt=-ffunction-sections \
+        --copt=-fdata-sections \
+        --copt=-fstack-protector \
+        --copt=-Oz \
+        --copt=-fomit-frame-pointer \
+        --copt=-DABSL_MIN_LOG_LEVEL=2 \
+        --linkopt=-Wl,--gc-sections,--strip-all \
         //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mediapipe_face_detection.aar
 
     # It should print:
@@ -117,7 +137,7 @@ each project.
         implementation 'com.google.flogger:flogger-system-backend:latest.release'
         implementation 'com.google.code.findbugs:jsr305:latest.release'
         implementation 'com.google.guava:guava:27.0.1-android'
-        implementation 'com.google.protobuf:protobuf-java:3.11.4'
+        implementation 'com.google.protobuf:protobuf-javalite:3.19.1'
         // CameraX core library
         def camerax_version = "1.0.0-beta10"
         implementation "androidx.camera:camera-core:$camerax_version"

docs/getting_started/android_solutions.md

@@ -1,58 +1,55 @@
 ---
 layout: default
-title: Android Solutions
+title: MediaPipe Android Solutions
 parent: MediaPipe on Android
 grand_parent: Getting Started
 nav_order: 2
 ---
 
-# Android Solution APIs
+# MediaPipe Android Solutions
 {: .no_toc }
 
 1. TOC
 {:toc}
 ---
 
-Please follow instructions below to use the MediaPipe Solution APIs in Android
-Studio projects and build the Android example apps in the supported MediaPipe
-[solutions](../solutions/solutions.md).
+MediaPipe Android Solution APIs (currently in alpha) are available in:
 
-## Integrate MediaPipe Android Solutions in Android Studio
+*   [MediaPipe Face Detection](../solutions/face_detection#android-solution-api)
+*   [MediaPipe Face Mesh](../solutions/face_mesh#android-solution-api)
+*   [MediaPipe Hands](../solutions/hands#android-solution-api)
 
-MediaPipe Android Solution APIs (currently in alpha) are now available in
+## Incorporation in Android Studio
+
+Prebuilt packages of Android Solution APIs can be found in
 [Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
-To incorporate MediaPipe Android Solutions into an Android Studio project, add
-the following into the project's Gradle dependencies:
+To incorporate them into an Android Studio project, add the following into the
+project's Gradle dependencies:
 
 ```
 dependencies {
-    // MediaPipe solution-core is the foundation of any MediaPipe solutions.
+    // MediaPipe solution-core is the foundation of any MediaPipe Solutions.
     implementation 'com.google.mediapipe:solution-core:latest.release'
-    // Optional: MediaPipe Hands solution.
-    implementation 'com.google.mediapipe:hands:latest.release'
-    // Optional: MediaPipe FaceMesh solution.
+    // Optional: MediaPipe Face Detection Solution.
+    implementation 'com.google.mediapipe:facedetection:latest.release'
+    // Optional: MediaPipe Face Mesh Solution.
     implementation 'com.google.mediapipe:facemesh:latest.release'
-    // MediaPipe deps
-    implementation 'com.google.flogger:flogger:latest.release'
-    implementation 'com.google.flogger:flogger-system-backend:latest.release'
-    implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-java:3.11.4'
-    // CameraX core library
-    def camerax_version = "1.0.0-beta10"
-    implementation "androidx.camera:camera-core:$camerax_version"
-    implementation "androidx.camera:camera-camera2:$camerax_version"
-    implementation "androidx.camera:camera-lifecycle:$camerax_version"
+    // Optional: MediaPipe Hands Solution.
+    implementation 'com.google.mediapipe:hands:latest.release'
 }
 ```
 
-See the detailed solutions API usage examples for different use cases in the
-solution example apps'
-[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
-If the prebuilt maven packages are not sufficient, building the MediaPipe
-Android archive library locally by following these
-[instructions](./android_archive_library.md).
+If you need further customization, instead of using the prebuilt maven packages
+consider building a MediaPipe Android Archive library locally from source by
+following these [instructions](./android_archive_library.md).
 
-## Build solution example apps in Android Studio
+## Building solution example apps
+
+Detailed usage examples of the Android Solution APIs can be found in the
+[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions)
+of the solution example apps.
+
+To build these apps:
 
 1.  Open Android Studio Arctic Fox on Linux, macOS, or Windows.
 
@@ -73,7 +70,62 @@ Android archive library locally by following these
 
 6.  (Optional) Run solutions on CPU.
 
-    MediaPipe solution example apps run the pipeline and the model inference on
-    GPU by default. If needed, for example to run the apps on Android Emulator,
-    set the `RUN_ON_GPU` boolean variable to `false` in the app's
-    MainActivity.java to run the pipeline and the model inference on CPU.
+    MediaPipe solution example apps run the pipeline and model inference on GPU
+    by default. If needed, for example to run the apps on Android Emulator, set
+    the `RUN_ON_GPU` boolean variable to `false` in the app's
+    `MainActivity.java` to run the pipeline and model inference on CPU.
+
+## MediaPipe Solution APIs Terms of Service
+
+Last modified: November 12, 2021
+
+Use of MediaPipe Solution APIs is subject to the
+[Google APIs Terms of Service](https://developers.google.com/terms),
+[Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy),
+and the terms below. Please check back from time to time as these terms and
+policies are occasionally updated.
+
+**Privacy**
+
+When you use MediaPipe Solution APIs, processing of the input data (e.g. images,
+video, text) fully happens on-device, and **MediaPipe does not send that input
+data to Google servers**. As a result, you can use our APIs for processing data
+that should not leave the device.
+
+MediaPipe Android Solution APIs will contact Google servers from time to time in
+order to receive things like bug fixes, updated models, and hardware accelerator
+compatibility information. MediaPipe Android Solution APIs also send metrics
+about the performance and utilization of the APIs in your app to Google. Google
+uses this metrics data to measure performance, API usage, debug, maintain and
+improve the APIs, and detect misuse or abuse, as further described in our
+[Privacy Policy](https://policies.google.com/privacy).
+
+**You are responsible for obtaining informed consent from your app users about
+Google’s processing of MediaPipe metrics data as required by applicable law.**
+
+Data we collect may include the following, across all MediaPipe Android Solution
+APIs:
+
+-   Device information (such as manufacturer, model, OS version and build) and
+    available ML hardware accelerators (GPU and DSP). Used for diagnostics and
+    usage analytics.
+
+-   App identification information (package name / bundle id, app version). Used
+    for diagnostics and usage analytics.
+
+-   API configuration (such as image format, resolution, and MediaPipe version
+    used). Used for diagnostics and usage analytics.
+
+-   Event type (such as initialize, download model, update, run, and detection).
+    Used for diagnostics and usage analytics.
+
+-   Error codes. Used for diagnostics.
+
+-   Performance metrics. Used for diagnostics.
+
+-   Per-installation identifiers that do not uniquely identify a user or
+    physical device. Used for operation of remote configuration and usage
+    analytics.
+
+-   Network request sender IP addresses. Used for remote configuration
+    diagnostics. Collected IP addresses are retained temporarily.

docs/getting_started/faq.md

@@ -103,7 +103,7 @@ monotonically increasing timestamps. By convention, realtime calculators and
 graphs use the recording time or the presentation time as the timestamp for each
 packet, with each timestamp representing microseconds since
 `Jan/1/1970:00:00:00`. This allows packets from various sources to be processed
-in a gloablly consistent order.
+in a globally consistent order.
 
 Normally for offline processing, every input packet is processed and processing
 continues as long as necessary. For online processing, it is often necessary to

docs/getting_started/gpu_support.md

@@ -59,6 +59,21 @@ OpenGL ES profile shading language version string: OpenGL ES GLSL ES 3.20
 OpenGL ES profile extensions:
 ```
 
+If you have connected to your computer through SSH and find when you probe for
+GPU information you see the output:
+
+```bash
+glxinfo | grep -i opengl
+Error: unable to open display
+```
+
+Try re-establishing your SSH connection with the `-X` option and try again. For
+example:
+
+```bash
+ssh -X <user>@<host>
+```
+
 *Notice the ES 3.20 text above.*
 
 You need to see ES 3.1 or greater printed in order to perform TFLite inference

docs/getting_started/hello_world_ios.md

@@ -113,6 +113,10 @@ bazel to build the iOS application. The content of the
 5.  `Main.storyboard` and `Launch.storyboard`
 6.  `Assets.xcassets` directory.
 
+Note: In newer versions of Xcode, you may see additional files `SceneDelegate.h`
+and `SceneDelegate.m`. Make sure to copy them too and add them to the `BUILD`
+file mentioned below.
+
 Copy these files to a directory named `HelloWorld` to a location that can access
 the MediaPipe source code. For example, the source code of the application that
 we will build in this tutorial is located in
@@ -127,7 +131,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
 rules:
 
 ```
-MIN_IOS_VERSION = "10.0"
+MIN_IOS_VERSION = "11.0"
 
 load(
     "@build_bazel_rules_apple//apple:ios.bzl",
@@ -247,6 +251,12 @@ We need to get frames from the `_cameraSource` into our application
 `MPPInputSourceDelegate`. So our application `ViewController` can be a delegate
 of `_cameraSource`.
 
+Update the interface definition of `ViewController` accordingly:
+
+```
+@interface ViewController () <MPPInputSourceDelegate>
+```
+
 To handle camera setup and process incoming frames, we should use a queue
 different from the main queue. Add the following to the implementation block of
 the `ViewController`:
@@ -288,6 +298,12 @@ utility called `MPPLayerRenderer` to display images on the screen. This utility
 can be used to display `CVPixelBufferRef` objects, which is the type of the
 images provided by `MPPCameraInputSource` to its delegates.
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPLayerRenderer.h"
+```
+
 To display images of the screen, we need to add a new `UIView` object called
 `_liveView` to the `ViewController`.
 
@@ -411,6 +427,12 @@ Objective-C++.
 
 ### Use the graph in `ViewController`
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPGraph.h"
+```
+
 Declare a static constant with the name of the graph, the input stream and the
 output stream:
 
@@ -549,6 +571,12 @@ method to receive packets on this output stream and display them on the screen:
 }
 ```
 
+Update the interface definition of `ViewController` with `MPPGraphDelegate`:
+
+```
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+```
+
 And that is all! Build and run the app on your iOS device. You should see the
 results of running the edge detection graph on a live video feed. Congrats!
 
@@ -560,5 +588,5 @@ appropriate `BUILD` file dependencies for the edge detection graph.
 
 [Bazel]:https://bazel.build/
 [`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
-[common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common)
-[helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld)
+[common]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common
+[helloworld]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld

docs/getting_started/install.md

@@ -569,7 +569,7 @@ next section.
 
     Option 1. Follow
     [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.7.2 or higher.
+    to install Bazel 5.0.0 or higher.
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
@@ -657,7 +657,7 @@ cameras. Alternatively, you use a video file as input.
 
     Note: Windows' and WSL’s adb versions must be the same version, e.g., if WSL
     has ADB 1.0.39, you need to download the corresponding Windows ADB from
-    [here](https://dl.google.com/android/repository/platform-tools_r26.0.1-windows.zip).
+    [here](https://dl.google.com/android/repository/platform-tools_r30.0.3-windows.zip).
 
 3.  Launch WSL.
 
@@ -796,7 +796,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
     ```bash
     $ docker run -it --name mediapipe mediapipe:latest
 
-    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
+    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world
 
     # Should print:
     # Hello World!

docs/getting_started/ios.md

@@ -32,9 +32,14 @@ example apps, start from, start from
     xcode-select --install
     ```
 
-3.  Install [Bazel](https://bazel.build/).
+3.  Install [Bazelisk](https://github.com/bazelbuild/bazelisk)
+.
 
-    We recommend using [Homebrew](https://brew.sh/) to get the latest version.
+    We recommend using [Homebrew](https://brew.sh/) to get the latest versions.
+
+    ```bash
+    brew install bazelisk
+    ```
 
 4.  Set Python 3.7 as the default Python version and install the Python "six"
     library. This is needed for TensorFlow.
@@ -187,6 +192,9 @@ Note: When you ask Xcode to run an app, by default it will use the Debug
 configuration. Some of our demos are computationally heavy; you may want to use
 the Release configuration for better performance.
 
+Note: Due to an imcoptibility caused by one of our dependencies, MediaPipe
+cannot be used for apps running on the iPhone Simulator on Apple Silicon (M1).
+
 Tip: To switch build configuration in Xcode, click on the target menu, choose
 "Edit Scheme...", select the Run action, and switch the Build Configuration from
 Debug to Release. Note that this is set independently for each target.

docs/getting_started/python_framework.md

@@ -126,6 +126,7 @@ following steps:
             }
             return packet.Get<MyType>();
           });
+    }
     }  // namespace mediapipe
     ```
 
@@ -249,12 +250,12 @@ three stages: initialization and setup, graph run, and graph shutdown.
     graph.start_run()
 
     graph.add_packet_to_input_stream(
-        'in_stream', mp.packet_creator.create_str('abc').at(0))
+        'in_stream', mp.packet_creator.create_string('abc').at(0))
 
     rgb_img = cv2.cvtColor(cv2.imread('/path/to/your/image.png'), cv2.COLOR_BGR2RGB)
     graph.add_packet_to_input_stream(
         'in_stream',
-        mp.packet_creator.create_image_frame(format=mp.ImageFormat.SRGB,
+        mp.packet_creator.create_image_frame(image_format=mp.ImageFormat.SRGB,
                                              data=rgb_img).at(1))
     ```
 

docs/getting_started/troubleshooting.md

@@ -108,14 +108,14 @@ ERROR: No matching distribution found for mediapipe
 
 after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system.
 Please note that MediaPipe Python PyPI officially supports the **64-bit**
-version of Python 3.7 and above on the following OS:
+version of Python 3.7 to 3.10 on the following OS:
 
 -   x86_64 Linux
 -   x86_64 macOS 10.15+
 -   amd64 Windows
 
 If the OS is currently supported and you still see this error, please make sure
-that both the Python and pip binary are for Python 3.7 and above. Otherwise,
+that both the Python and pip binary are for Python 3.7 to 3.10. Otherwise,
 please consider building the MediaPipe Python package locally by following the
 instructions [here](python.md#building-mediapipe-python-package).
 

docs/index.md

@@ -136,8 +136,8 @@ run code search using
 
 ## Community
 
-*   [Awesome MediaPipe](https://mediapipe.org) - A curated list of awesome
-    MediaPipe related frameworks, libraries and software
+*   [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
+    curated list of awesome MediaPipe related frameworks, libraries and software
 *   [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
 *   [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
     community discussion around MediaPipe

docs/solutions/face_detection.md

@@ -26,7 +26,7 @@ MediaPipe Face Detection is an ultrafast face detection solution that comes with
 face detector tailored for mobile GPU inference. The detector's super-realtime
 performance enables it to be applied to any live viewfinder experience that
 requires an accurate facial region of interest as an input for other
-task-specific models, such as 3D facial keypoint or geometry estimation (e.g.,
+task-specific models, such as 3D facial keypoint estimation (e.g.,
 [MediaPipe Face Mesh](./face_mesh.md)), facial features or expression
 classification, and face region segmentation. BlazeFace uses a lightweight
 feature extraction network inspired by, but distinct from
@@ -121,12 +121,10 @@ with mp_face_detection.FaceDetection(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_detection.process(image)
 
     # Draw the face detection annotations on the image.
@@ -135,7 +133,8 @@ with mp_face_detection.FaceDetection(
     if results.detections:
       for detection in results.detections:
         mp_drawing.draw_detection(image, detection)
-    cv2.imshow('MediaPipe Face Detection', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Detection', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -200,7 +199,7 @@ const faceDetection = new FaceDetection({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
 }});
 faceDetection.setOptions({
-  modelSelection: 0
+  modelSelection: 0,
   minDetectionConfidence: 0.5
 });
 faceDetection.onResults(onResults);
@@ -216,6 +215,214 @@ camera.start();
 </script>
 ```
 
+### Android Solution API
+
+Please first follow general
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection),
+and learn more in the usage example below.
+
+Supported configuration options:
+
+*   [staticImageMode](#static_image_mode)
+*   [modelSelection](#model_selection)
+
+#### Camera Input
+
+```java
+// For camera input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Detection Solution.
+CameraInput cameraInput = new CameraInput(this);
+cameraInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
+      RelativeKeypoint noseTip =
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+// The runnable to start camera after the GLSurfaceView is attached.
+glSurfaceView.post(
+    () ->
+        cameraInput.start(
+            this,
+            faceDetection.getGlContext(),
+            CameraInput.CameraFacing.FRONT,
+            glSurfaceView.getWidth(),
+            glSurfaceView.getHeight()));
+```
+
+#### Image Input
+
+```java
+// For reading images from gallery and drawing the output in an ImageView.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(true)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+
+// Connects MediaPipe Face Detection Solution to the user-defined ImageView
+// instance that allows users to have the custom drawing of the output landmarks
+// on it. See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java
+// as an example.
+FaceDetectionResultImageView imageView = new FaceDetectionResultImageView(this);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
+      int width = faceDetectionResult.inputBitmap().getWidth();
+      int height = faceDetectionResult.inputBitmap().getHeight();
+      RelativeKeypoint noseTip =
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
+              noseTip.getX() * width, noseTip.getY() * height));
+      // Request canvas drawing.
+      imageView.setFaceDetectionResult(faceDetectionResult);
+      runOnUiThread(() -> imageView.update());
+    });
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// ActivityResultLauncher to get an image from the gallery as Bitmap.
+ActivityResultLauncher<Intent> imageGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null && result.getResultCode() == RESULT_OK) {
+            Bitmap bitmap = null;
+            try {
+              bitmap =
+                  MediaStore.Images.Media.getBitmap(
+                      this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
+            } catch (IOException e) {
+              Log.e(TAG, "Bitmap reading error:" + e);
+            }
+            if (bitmap != null) {
+              faceDetection.send(bitmap);
+            }
+          }
+        });
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
+```
+
+#### Video Input
+
+```java
+// For video input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Detection Solution.
+VideoInput videoInput = new VideoInput(this);
+videoInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
+      RelativeKeypoint noseTip =
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+ActivityResultLauncher<Intent> videoGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null) {
+            if (result.getResultCode() == RESULT_OK) {
+              glSurfaceView.post(
+                  () ->
+                      videoInput.start(
+                          this,
+                          resultIntent.getData(),
+                          faceDetection.getGlContext(),
+                          glSurfaceView.getWidth(),
+                          glSurfaceView.getHeight()));
+            }
+          }
+        });
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
+```
+
 ## Example Apps
 
 Please first see general instructions for

docs/solutions/face_mesh.md

@@ -20,34 +20,34 @@ nav_order: 2
 
 ## Overview
 
-MediaPipe Face Mesh is a face geometry solution that estimates 468 3D face
-landmarks in real-time even on mobile devices. It employs machine learning (ML)
-to infer the 3D surface geometry, requiring only a single camera input without
-the need for a dedicated depth sensor. Utilizing lightweight model architectures
-together with GPU acceleration throughout the pipeline, the solution delivers
-real-time performance critical for live experiences.
+MediaPipe Face Mesh is a solution that estimates 468 3D face landmarks in
+real-time even on mobile devices. It employs machine learning (ML) to infer the
+3D facial surface, requiring only a single camera input without the need for a
+dedicated depth sensor. Utilizing lightweight model architectures together with
+GPU acceleration throughout the pipeline, the solution delivers real-time
+performance critical for live experiences.
 
-Additionally, the solution is bundled with the Face Geometry module that bridges
-the gap between the face landmark estimation and useful real-time augmented
-reality (AR) applications. It establishes a metric 3D space and uses the face
-landmark screen positions to estimate face geometry within that space. The face
-geometry data consists of common 3D geometry primitives, including a face pose
-transformation matrix and a triangular face mesh. Under the hood, a lightweight
-statistical analysis method called
+Additionally, the solution is bundled with the Face Transform module that
+bridges the gap between the face landmark estimation and useful real-time
+augmented reality (AR) applications. It establishes a metric 3D space and uses
+the face landmark screen positions to estimate a face transform within that
+space. The face transform data consists of common 3D primitives, including a
+face pose transformation matrix and a triangular face mesh. Under the hood, a
+lightweight statistical analysis method called
 [Procrustes Analysis](https://en.wikipedia.org/wiki/Procrustes_analysis) is
 employed to drive a robust, performant and portable logic. The analysis runs on
 CPU and has a minimal speed/memory footprint on top of the ML model inference.
 
 ![face_mesh_ar_effects.gif](../images/face_mesh_ar_effects.gif) |
 :-------------------------------------------------------------: |
-*Fig 1. AR effects utilizing facial surface geometry.*          |
+*Fig 1. AR effects utilizing the 3D facial surface.*            |
 
 ## ML Pipeline
 
 Our ML pipeline consists of two real-time deep neural network models that work
 together: A detector that operates on the full image and computes face locations
 and a 3D face landmark model that operates on those locations and predicts the
-approximate surface geometry via regression. Having the face accurately cropped
+approximate 3D surface via regression. Having the face accurately cropped
 drastically reduces the need for common data augmentations like affine
 transformations consisting of rotations, translation and scale changes. Instead
 it allows the network to dedicate most of its capacity towards coordinate
@@ -55,8 +55,8 @@ prediction accuracy. In addition, in our pipeline the crops can also be
 generated based on the face landmarks identified in the previous frame, and only
 when the landmark model could no longer identify face presence is the face
 detector invoked to relocalize the face. This strategy is similar to that
-employed in our [MediaPipe Hands](./hands.md) solution, which uses a palm detector
-together with a hand landmark model.
+employed in our [MediaPipe Hands](./hands.md) solution, which uses a palm
+detector together with a hand landmark model.
 
 The pipeline is implemented as a MediaPipe
 [graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
@@ -111,7 +111,24 @@ You can find more information about the face landmark model in this
 :------------------------------------------------------------------------: |
 *Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
 
-## Face Geometry Module
+#### Attention Mesh Model
+
+In addition to the [Face Landmark Model](#face-landmark-model) we provide
+another model that applies
+[attention](https://en.wikipedia.org/wiki/Attention_(machine_learning)) to
+semantically meaningful face regions, and therefore predicting landmarks more
+accurately around lips, eyes and irises, at the expense of more compute. It
+enables applications like AR makeup and AR puppeteering.
+
+The attention mesh model can be selected in the Solution APIs via the
+[refine_landmarks](#refine_landmarks) option. You can also find more information
+about the model in this [paper](https://arxiv.org/abs/2006.10962).
+
+![attention_mesh_architecture.png](../images/attention_mesh_architecture.png) |
+:---------------------------------------------------------------------------: |
+*Fig 3. Attention Mesh: Overview of model architecture.*                      |
+
+## Face Transform Module
 
 The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
 detection in the screen coordinate space: the X- and Y- coordinates are
@@ -123,7 +140,7 @@ enable the full spectrum of augmented reality (AR) features like aligning a
 virtual 3D object with a detected face.
 
 The
-[Face Geometry module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
+[Face Transform module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
 moves away from the screen coordinate space towards a metric 3D space and
 provides necessary primitives to handle a detected face as a regular 3D object.
 By design, you'll be able to use a perspective camera to project the final 3D
@@ -134,7 +151,7 @@ landmark positions are not changed.
 
 #### Metric 3D Space
 
-The **Metric 3D space** established within the Face Geometry module is a
+The **Metric 3D space** established within the Face Transform module is a
 right-handed orthonormal metric 3D coordinate space. Within the space, there is
 a **virtual perspective camera** located at the space origin and pointed in the
 negative direction of the Z-axis. In the current pipeline, it is assumed that
@@ -145,8 +162,8 @@ be set freely, however for better results it is advised to set them as close to
 the *real physical camera parameters* as possible.
 
 ![face_geometry_metric_3d_space.gif](../images/face_geometry_metric_3d_space.gif) |
-:----------------------------------------------------------------------------: |
-*Fig 3. A visualization of multiple key elements in the Metric 3D space.*      |
+:-------------------------------------------------------------------------------: |
+*Fig 4. A visualization of multiple key elements in the Metric 3D space.*         |
 
 #### Canonical Face Model
 
@@ -167,11 +184,11 @@ functions:
 
 ### Components
 
-#### Geometry Pipeline
+#### Transform Pipeline
 
-The **Geometry Pipeline** is a key component, which is responsible for
-estimating face geometry objects within the Metric 3D space. On each frame, the
-following steps are executed in the given order:
+The **Transform Pipeline** is a key component, which is responsible for
+estimating the face transform objects within the Metric 3D space. On each frame,
+the following steps are executed in the given order:
 
 -   Face landmark screen coordinates are converted into the Metric 3D space
     coordinates;
@@ -182,12 +199,12 @@ following steps are executed in the given order:
     positions (XYZ), while both the vertex texture coordinates (UV) and the
     triangular topology are inherited from the canonical face model.
 
-The geometry pipeline is implemented as a MediaPipe
+The transform pipeline is implemented as a MediaPipe
 [calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
-For your convenience, the face geometry pipeline calculator is bundled together
-with corresponding metadata into a unified MediaPipe
+For your convenience, this calculator is bundled together with corresponding
+metadata into a unified MediaPipe
 [subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt).
-The face geometry format is defined as a Protocol Buffer
+The face transform format is defined as a Protocol Buffer
 [message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
 
 #### Effect Renderer
@@ -210,7 +227,7 @@ The effect renderer is implemented as a MediaPipe
 
 | ![face_geometry_renderer.gif](../images/face_geometry_renderer.gif)     |
 | :---------------------------------------------------------------------: |
-| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
+| *Fig 5. An example of face effects rendered by the Face Transform Effect Renderer.* |
 
 ## Solution APIs
 
@@ -234,6 +251,12 @@ unrelated, images. Default to `false`.
 
 Maximum number of faces to detect. Default to `1`.
 
+#### refine_landmarks
+
+Whether to further refine the landmark coordinates around the eyes and lips, and
+output additional landmarks around the irises by applying the
+[Attention Mesh Model](#attention-mesh-model). Default to `false`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
@@ -271,6 +294,7 @@ Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
 *   [max_num_faces](#max_num_faces)
+*   [refine_landmarks](#refine_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -287,6 +311,7 @@ drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 with mp_face_mesh.FaceMesh(
     static_image_mode=True,
     max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5) as face_mesh:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
@@ -313,12 +338,21 @@ with mp_face_mesh.FaceMesh(
           landmark_drawing_spec=None,
           connection_drawing_spec=mp_drawing_styles
           .get_default_face_mesh_contours_style())
+      mp_drawing.draw_landmarks(
+          image=annotated_image,
+          landmark_list=face_landmarks,
+          connections=mp_face_mesh.FACEMESH_IRISES,
+          landmark_drawing_spec=None,
+          connection_drawing_spec=mp_drawing_styles
+          .get_default_face_mesh_iris_connections_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
 drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 cap = cv2.VideoCapture(0)
 with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5,
     min_tracking_confidence=0.5) as face_mesh:
   while cap.isOpened():
@@ -328,12 +362,10 @@ with mp_face_mesh.FaceMesh(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_mesh.process(image)
 
     # Draw the face mesh annotations on the image.
@@ -355,7 +387,15 @@ with mp_face_mesh.FaceMesh(
             landmark_drawing_spec=None,
             connection_drawing_spec=mp_drawing_styles
             .get_default_face_mesh_contours_style())
-    cv2.imshow('MediaPipe FaceMesh', image)
+        mp_drawing.draw_landmarks(
+            image=image,
+            landmark_list=face_landmarks,
+            connections=mp_face_mesh.FACEMESH_IRISES,
+            landmark_drawing_spec=None,
+            connection_drawing_spec=mp_drawing_styles
+            .get_default_face_mesh_iris_connections_style())
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Mesh', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -370,6 +410,7 @@ and the following usage example.
 Supported configuration options:
 
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -410,8 +451,10 @@ function onResults(results) {
                      {color: '#C0C0C070', lineWidth: 1});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_IRIS, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_IRIS, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
     }
@@ -424,6 +467,7 @@ const faceMesh = new FaceMesh({locateFile: (file) => {
 }});
 faceMesh.setOptions({
   maxNumFaces: 1,
+  refineLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });
@@ -443,18 +487,16 @@ camera.start();
 ### Android Solution API
 
 Please first follow general
-[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the FaceMash solution API in the
-companion
-[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
-following
-[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh),
 and learn more in the usage example below.
 
 Supported configuration options:
 
 *   [staticImageMode](#static_image_mode)
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   runOnGpu: Run the pipeline and the model inference on GPU or CPU.
 
 #### Camera Input
@@ -463,17 +505,18 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Mesh Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -481,18 +524,18 @@ cameraInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -504,7 +547,7 @@ glSurfaceView.post(
     () ->
         cameraInput.start(
             this,
-            facemesh.getGlContext(),
+            faceMesh.getGlContext(),
             CameraInput.CameraFacing.FRONT,
             glSurfaceView.getWidth(),
             glSurfaceView.getHeight()));
@@ -516,17 +559,18 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumFaces(1)                           // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
 
-// Connects MediaPipe FaceMesh to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Face Mesh Solution to the user-defined ImageView instance
+// that allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
 // as an example.
 FaceMeshResultImageView imageView = new FaceMeshResultImageView(this);
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       int width = faceMeshResult.inputBitmap().getWidth();
       int height = faceMeshResult.inputBitmap().getHeight();
@@ -535,14 +579,14 @@ facemesh.setResultListener(
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
+              "MediaPipe Face Mesh nose coordinates (pixel values): x=%f, y=%f",
               noseLandmark.getX() * width, noseLandmark.getY() * height));
       // Request canvas drawing.
       imageView.setFaceMeshResult(faceMeshResult);
       runOnUiThread(() -> imageView.update());
     });
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
 // ActivityResultLauncher to get an image from the gallery as Bitmap.
 ActivityResultLauncher<Intent> imageGetter =
@@ -556,17 +600,18 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
             if (bitmap != null) {
-              facemesh.send(bitmap);
+              faceMesh.send(bitmap);
             }
           }
         });
-Intent gallery = new Intent(
-    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-imageGetter.launch(gallery);
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
 ```
 
 #### Video Input
@@ -575,17 +620,18 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Mesh Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -593,18 +639,18 @@ videoInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -623,15 +669,15 @@ ActivityResultLauncher<Intent> videoGetter =
                       videoInput.start(
                           this,
                           resultIntent.getData(),
-                          facemesh.getGlContext(),
+                          faceMesh.getGlContext(),
                           glSurfaceView.getWidth(),
                           glSurfaceView.getHeight()));
             }
           }
         });
-Intent gallery =
-    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-videoGetter.launch(gallery);
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
 ```
 
 ## Example Apps

docs/solutions/hands.md

@@ -91,8 +91,10 @@ To detect initial hand locations, we designed a
 mobile real-time uses in a manner similar to the face detection model in
 [MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
 task: our
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite)
-has to work across a variety of hand sizes with a large scale span (~20x)
+[lite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite)
+and
+[full model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite)
+have to work across a variety of hand sizes with a large scale span (~20x)
 relative to the image frame and be able to detect occluded and self-occluded
 hands. Whereas faces have high contrast patterns, e.g., in the eye and mouth
 region, the lack of such features in hands makes it comparatively difficult to
@@ -120,7 +122,7 @@ just 86.22%.
 ### Hand Landmark Model
 
 After the palm detection over the whole image our subsequent hand landmark
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite)
+[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite)
 performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
 the detected hand regions via regression, that is direct coordinate prediction.
 The model learns a consistent internal hand pose representation and is robust
@@ -163,6 +165,11 @@ unrelated, images. Default to `false`.
 
 Maximum number of hands to detect. Default to `2`.
 
+#### model_complexity
+
+Complexity of the hand landmark model: `0` or `1`. Landmark accuracy as well as
+inference latency generally go up with the model complexity. Default to `1`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the hand detection model for the
@@ -190,6 +197,13 @@ of 21 hand landmarks and each landmark is composed of `x`, `y` and `z`. `x` and
 and the smaller the value the closer the landmark is to the camera. The
 magnitude of `z` uses roughly the same scale as `x`.
 
+#### multi_hand_world_landmarks
+
+Collection of detected/tracked hands, where each hand is represented as a list
+of 21 hand landmarks in world coordinates. Each landmark is composed of `x`, `y`
+and `z`: real-world 3D coordinates in meters with the origin at the hand's
+approximate geometric center.
+
 #### multi_handedness
 
 Collection of handedness of the detected/tracked hands (i.e. is it a left or
@@ -212,6 +226,7 @@ Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
 *   [max_num_hands](#max_num_hands)
+*   [model_complexity](#model_complexity)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -256,10 +271,17 @@ with mp_hands.Hands(
           mp_drawing_styles.get_default_hand_connections_style())
     cv2.imwrite(
         '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
+    # Draw hand world landmarks.
+    if not results.multi_hand_world_landmarks:
+      continue
+    for hand_world_landmarks in results.multi_hand_world_landmarks:
+      mp_drawing.plot_landmarks(
+        hand_world_landmarks, mp_hands.HAND_CONNECTIONS, azimuth=5)
 
 # For webcam input:
 cap = cv2.VideoCapture(0)
 with mp_hands.Hands(
+    model_complexity=0,
     min_detection_confidence=0.5,
     min_tracking_confidence=0.5) as hands:
   while cap.isOpened():
@@ -269,12 +291,10 @@ with mp_hands.Hands(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = hands.process(image)
 
     # Draw the hand annotations on the image.
@@ -288,7 +308,8 @@ with mp_hands.Hands(
             mp_hands.HAND_CONNECTIONS,
             mp_drawing_styles.get_default_hand_landmarks_style(),
             mp_drawing_styles.get_default_hand_connections_style())
-    cv2.imshow('MediaPipe Hands', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Hands', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -303,6 +324,7 @@ and a [fun application], and the following usage example.
 Supported configuration options:
 
 *   [maxNumHands](#max_num_hands)
+*   [modelComplexity](#model_complexity)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -352,6 +374,7 @@ const hands = new Hands({locateFile: (file) => {
 }});
 hands.setOptions({
   maxNumHands: 2,
+  modelComplexity: 1,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });
@@ -371,13 +394,10 @@ camera.start();
 ### Android Solution API
 
 Please first follow general
-[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Hands solution API in the
-companion
-[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
-following
-[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
-and learn more in usage example below.
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands),
+and learn more in the usage example below.
 
 Supported configuration options:
 
@@ -391,14 +411,14 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
+// Initializes a new CameraInput instance and connects it to MediaPipe Hands Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));
@@ -415,8 +435,11 @@ glSurfaceView.setRenderInputImage(true);
 
 hands.setResultListener(
     handsResult -> {
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -444,22 +467,25 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumHands(1)                        // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 
-// Connects MediaPipe Hands to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Hands Solution to the user-defined ImageView instance that
+// allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
 // as an example.
 HandsResultImageView imageView = new HandsResultImageView(this);
 hands.setResultListener(
     handsResult -> {
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
       int width = handsResult.inputBitmap().getWidth();
       int height = handsResult.inputBitmap().getHeight();
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -484,6 +510,7 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
@@ -492,9 +519,9 @@ ActivityResultLauncher<Intent> imageGetter =
             }
           }
         });
-Intent gallery = new Intent(
-    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-imageGetter.launch(gallery);
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
 ```
 
 #### Video Input
@@ -503,14 +530,14 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
+// Initializes a new VideoInput instance and connects it to MediaPipe Hands Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));
@@ -527,8 +554,11 @@ glSurfaceView.setRenderInputImage(true);
 
 hands.setResultListener(
     handsResult -> {
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -557,9 +587,9 @@ ActivityResultLauncher<Intent> videoGetter =
             }
           }
         });
-Intent gallery =
-    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-videoGetter.launch(gallery);
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
 ```
 
 ## Example Apps

docs/solutions/holistic.md

@@ -147,6 +147,23 @@ If set to `true`, the solution filters pose landmarks across different input
 images to reduce jitter, but ignored if [static_image_mode](#static_image_mode)
 is also set to `true`. Default to `true`.
 
+#### enable_segmentation
+
+If set to `true`, in addition to the pose, face and hand landmarks the solution
+also generates the segmentation mask. Default to `false`.
+
+#### smooth_segmentation
+
+If set to `true`, the solution filters segmentation masks across different input
+images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
+is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
+`true`.
+
+#### refine_face_landmarks
+
+Whether to further refine the landmark coordinates around the eyes and lips, and
+output additional landmarks around the irises. Default to `false`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
@@ -207,6 +224,15 @@ the camera. The magnitude of `z` uses roughly the same scale as `x`.
 A list of 21 hand landmarks on the right hand, in the same representation as
 [left_hand_landmarks](#left_hand_landmarks).
 
+#### segmentation_mask
+
+The output segmentation mask, predicted only when
+[enable_segmentation](#enable_segmentation) is set to `true`. The mask has the
+same width and height as the input image, and contains values in `[0.0, 1.0]`
+where `1.0` and `0.0` indicate high certainty of a "human" and "background"
+pixel respectively. Please refer to the platform-specific usage examples below
+for usage details.
+
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
@@ -218,6 +244,9 @@ Supported configuration options:
 *   [static_image_mode](#static_image_mode)
 *   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
+*   [enable_segmentation](#enable_segmentation)
+*   [smooth_segmentation](#smooth_segmentation)
+*   [refine_face_landmarks](#refine_face_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -232,7 +261,9 @@ mp_holistic = mp.solutions.holistic
 IMAGE_FILES = []
 with mp_holistic.Holistic(
     static_image_mode=True,
-    model_complexity=2) as holistic:
+    model_complexity=2,
+    enable_segmentation=True,
+    refine_face_landmarks=True) as holistic:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -245,8 +276,16 @@ with mp_holistic.Holistic(
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
       )
-    # Draw pose, left and right hands, and face landmarks on the image.
+
     annotated_image = image.copy()
+    # Draw segmentation on the image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
+    bg_image = np.zeros(image.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    annotated_image = np.where(condition, annotated_image, bg_image)
+    # Draw pose, left and right hands, and face landmarks on the image.
     mp_drawing.draw_landmarks(
         annotated_image,
         results.face_landmarks,
@@ -277,12 +316,10 @@ with mp_holistic.Holistic(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = holistic.process(image)
 
     # Draw landmark annotation on the image.
@@ -301,7 +338,8 @@ with mp_holistic.Holistic(
         mp_holistic.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles
         .get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Holistic', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Holistic', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -317,6 +355,9 @@ Supported configuration options:
 
 *   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
+*   [enableSegmentation](#enable_segmentation)
+*   [smoothSegmentation](#smooth_segmentation)
+*   [refineFaceLandmarks](#refineFaceLandmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -349,8 +390,20 @@ const canvasCtx = canvasElement.getContext('2d');
 function onResults(results) {
   canvasCtx.save();
   canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(results.segmentationMask, 0, 0,
+                      canvasElement.width, canvasElement.height);
+
+  // Only overwrite existing pixels.
+  canvasCtx.globalCompositeOperation = 'source-in';
+  canvasCtx.fillStyle = '#00FF00';
+  canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
+
+  // Only overwrite missing pixels.
+  canvasCtx.globalCompositeOperation = 'destination-atop';
   canvasCtx.drawImage(
       results.image, 0, 0, canvasElement.width, canvasElement.height);
+
+  canvasCtx.globalCompositeOperation = 'source-over';
   drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
                  {color: '#00FF00', lineWidth: 4});
   drawLandmarks(canvasCtx, results.poseLandmarks,
@@ -374,6 +427,9 @@ const holistic = new Holistic({locateFile: (file) => {
 holistic.setOptions({
   modelComplexity: 1,
   smoothLandmarks: true,
+  enableSegmentation: true,
+  smoothSegmentation: true,
+  refineFaceLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });

docs/solutions/models.md

@@ -41,7 +41,10 @@ one over the other.
 *   Face landmark model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   [Model card](https://mediapipe.page.link/facemesh-mc)
+*   Face landmark model w/ attention (aka Attention Mesh):
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite)
+*   [Model card](https://mediapipe.page.link/facemesh-mc),
+    [Model card (w/ attention)](https://mediapipe.page.link/attentionmesh-mc)
 
 ### [Iris](https://google.github.io/mediapipe/solutions/iris)
 
@@ -52,13 +55,14 @@ one over the other.
 ### [Hands](https://google.github.io/mediapipe/solutions/hands)
 
 *   Palm detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite),
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite),
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
 *   Hand landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
-    [TFLite model (sparse)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_sparse.tflite),
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_lite.tflite),
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-*   [Model card](https://mediapipe.page.link/handmc), [Model card (sparse)](https://mediapipe.page.link/handmc-sparse)
+*   [Model card](https://mediapipe.page.link/handmc)
 
 ### [Pose](https://google.github.io/mediapipe/solutions/pose)
 

docs/solutions/object_detection.md

@@ -116,7 +116,7 @@ on how to build MediaPipe examples.
 
     Note: The following runs TensorFlow inference on CPU. If you would like to
     run inference on GPU (Linux only), please follow
-    [TensorFlow CUDA Support and Setup on Linux Desktop](gpu.md#tensorflow-cuda-support-and-setup-on-linux-desktop)
+    [TensorFlow CUDA Support and Setup on Linux Desktop](../getting_started/gpu_support.md#tensorflow-cuda-support-and-setup-on-linux-desktop)
     instead.
 
     To build the TensorFlow CPU inference example on desktop, run:

docs/solutions/objectron.md

@@ -338,11 +338,10 @@ with mp_objectron.Objectron(static_image_mode=False,
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Convert the BGR image to RGB.
-    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = objectron.process(image)
 
     # Draw the box landmarks on the image.
@@ -354,7 +353,8 @@ with mp_objectron.Objectron(static_image_mode=False,
               image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
             mp_drawing.draw_axis(image, detected_object.rotation,
                                  detected_object.translation)
-    cv2.imshow('MediaPipe Objectron', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Objectron', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -384,7 +384,7 @@ Supported configuration options:
   <meta charset="utf-8">
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/control_utils_3d.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils_3d/control_utils_3d.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/objectron/objectron.js" crossorigin="anonymous"></script>
 </head>
@@ -529,7 +529,7 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > ```
 > and then run
 >
-> ```build
+> ```bash
 > bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
 > ```
 > INPUT_DIR should be the folder with initial asset .obj files to be processed,

docs/solutions/pose.md

@@ -88,11 +88,11 @@ from [COCO topology](https://cocodataset.org/#keypoints-2020).
 
 Method                                                                                                | Yoga <br/> [`mAP`] | Yoga <br/> [`[email protected]`] | Dance <br/> [`mAP`] | Dance <br/> [`[email protected]`] | HIIT <br/> [`mAP`] | HIIT <br/> [`[email protected]`]
 ----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------:
-BlazePose.Heavy                                                                                       | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
-BlazePose.Full                                                                                        | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
-BlazePose.Lite                                                                                        | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
-[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
-[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
+BlazePose GHUM Heavy                                                                                  | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
+BlazePose GHUM Full                                                                                   | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
+BlazePose GHUM Lite                                                                                   | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
+[AlphaPose ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
+[Apple Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
 
 ![pose_tracking_pck_chart.png](../images/mobile/pose_tracking_pck_chart.png) |
 :--------------------------------------------------------------------------: |
@@ -101,11 +101,11 @@ BlazePose.Lite
 We designed our models specifically for live perception use cases, so all of
 them work in real-time on the majority of modern devices.
 
-Method          | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
---------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
-BlazePose.Heavy | 53 ms                                                                                        | 38 ms
-BlazePose.Full  | 25 ms                                                                                        | 27 ms
-BlazePose.Lite  | 20 ms                                                                                        | 25 ms
+Method               | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
+-------------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
+BlazePose GHUM Heavy | 53 ms                                                                                        | 38 ms
+BlazePose GHUM Full  | 25 ms                                                                                        | 27 ms
+BlazePose GHUM Lite  | 20 ms                                                                                        | 25 ms
 
 ## Models
 
@@ -125,7 +125,7 @@ hip midpoints.
 :----------------------------------------------------------------------------------------------------: |
 *Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
 
-### Pose Landmark Model (BlazePose GHUM 3D)
+### Pose Landmark Model (BlazePose [GHUM](https://github.com/google-research/google-research/tree/master/ghum) 3D)
 
 The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
 (see figure below).
@@ -141,7 +141,7 @@ Optionally, MediaPipe Pose can predicts a full-body
 Please find more detail in the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
 this [paper](https://arxiv.org/abs/2006.10204),
-[the model card](./models.md#pose) and the [Output](#Output) section below.
+[the model card](./models.md#pose) and the [Output](#output) section below.
 
 ## Solution APIs
 
@@ -237,7 +237,7 @@ pixel respectively. Please refer to the platform-specific usage examples below
 for usage details.
 
 *Fig 6. Example of MediaPipe Pose segmentation mask.* |
-:-----------------------------------------------------------: |
+:---------------------------------------------------: |
 <video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
 
 ### Python Solution API
@@ -281,8 +281,8 @@ with mp_pose.Pose(
       continue
     print(
         f'Nose coordinates: ('
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].x * image_width}, '
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].y * image_height})'
     )
 
     annotated_image = image.copy()
@@ -316,12 +316,10 @@ with mp_pose.Pose(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = pose.process(image)
 
     # Draw the pose annotation on the image.
@@ -332,7 +330,8 @@ with mp_pose.Pose(
         results.pose_landmarks,
         mp_pose.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Pose', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Pose', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -360,7 +359,7 @@ Supported configuration options:
   <meta charset="utf-8">
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/control_utils_3d.js" crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils_3d/control_utils_3d.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
   <script src="https://cdn.jsdelivr.net/npm/@mediapipe/pose/pose.js" crossorigin="anonymous"></script>
 </head>
@@ -369,6 +368,7 @@ Supported configuration options:
   <div class="container">
     <video class="input_video"></video>
     <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+    <div class="landmark-grid-container"></div>
   </div>
 </body>
 </html>
@@ -486,6 +486,7 @@ on how to build MediaPipe examples.
     [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
     ([presentation](https://youtu.be/YPpUOTRn5tA))
 *   [Models and model cards](./models.md#pose)
+*   [GHUM & GHUML: Generative 3D Human Shape and Articulated Pose Models](https://github.com/google-research/google-research/tree/master/ghum)
 *   [Web demo](https://code.mediapipe.dev/codepen/pose)
 *   [Python Colab](https://mediapipe.page.link/pose_py_colab)
 

docs/solutions/selfie_segmentation.md

@@ -262,7 +262,7 @@ to visualize its associated subgraphs, please see
     [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1DoeyGzMmWUsjfVgZfGGecrn7GKzYcEAo/view?usp=sharing)
     [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu:selfiesegmentationgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD)
 *   iOS target:
-    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](http:/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
+    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
 
 ### Desktop
 

docs/solutions/solutions.md

@@ -13,6 +13,9 @@ has_toc: false
 {:toc}
 ---
 
+MediaPipe offers open source cross-platform, customizable ML solutions for live
+and streaming media.
+
 <!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
 <!-- Whenever this table is updated, paste a copy to ../external_index.md. -->
 

docs/tools/tracing_and_profiling.md

@@ -258,13 +258,14 @@ Many of the following settings are advanced and not recommended for general
 usage. Consult [Enabling tracing and profiling](#enabling-tracing-and-profiling)
 for a friendlier introduction.
 
-histogram_interval_size_usec :Specifies the size of the runtimes histogram
-intervals (in microseconds) to generate the histogram of the Process() time. The
-last interval extends to +inf. If not specified, the interval is 1000000 usec =
-1 sec.
+histogram_interval_size_usec
+:   Specifies the size of the runtimes histogram intervals (in microseconds) to
+    generate the histogram of the `Process()` time. The last interval extends to
+    +inf. If not specified, the interval is 1000000 usec = 1 sec.
 
-num_histogram_intervals :Specifies the number of intervals to generate the
-histogram of the `Process()` runtime. If not specified, one interval is used.
+num_histogram_intervals
+:   Specifies the number of intervals to generate the histogram of the
+    `Process()` runtime. If not specified, one interval is used.
 
 enable_profiler
 :   If true, the profiler starts profiling when graph is initialized.
@@ -288,7 +289,7 @@ trace_event_types_disabled
 
 trace_log_path
 :   The output directory and base-name prefix for trace log files. Log files are
-    written to: StrCat(trace_log_path, index, "`.binarypb`")
+    written to: `StrCat(trace_log_path, index, ".binarypb")`
 
 trace_log_count
 :   The number of trace log files retained. The trace log files are named
@@ -310,8 +311,8 @@ trace_log_instant_events
 
 trace_log_interval_count
 :   The number of trace log intervals per file. The total log duration is:
-    `trace_log_interval_usec * trace_log_file_count * trace_log_interval_count`.
-    The default value specifies 10 intervals per file.
+    `trace_log_interval_usec * trace_log_count * trace_log_interval_count`. The
+    default value specifies 10 intervals per file.
 
 trace_log_disabled
 :   An option to turn ON/OFF writing trace files to disk. Saving trace files to

mediapipe/BUILD

@@ -75,6 +75,7 @@ alias(
     actual = select({
         ":macos_i386": ":macos_i386",
         ":macos_x86_64": ":macos_x86_64",
+        ":macos_arm64": ":macos_arm64",
         "//conditions:default": ":macos_i386",  # Arbitrarily chosen from above.
     }),
     visibility = ["//visibility:public"],
@@ -119,6 +120,15 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "macos_arm64",
+    values = {
+        "apple_platform_type": "macos",
+        "cpu": "darwin_arm64",
+    },
+    visibility = ["//visibility:public"],
+)
+
 [
     config_setting(
         name = arch,

mediapipe/calculators/audio/BUILD

@@ -244,6 +244,7 @@ cc_test(
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:test_util",
         "@com_google_absl//absl/flags:flag",
     ],
 )

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -20,8 +20,12 @@
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/test_util.h"
 
 namespace mediapipe {
+namespace {
+
+constexpr char kTestPackageRoot[] = "mediapipe/calculators/audio";
 
 TEST(AudioDecoderCalculatorTest, TestWAV) {
   CalculatorGraphConfig::Node node_config =
@@ -37,9 +41,8 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
         })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
-      file::JoinPath("./",
-                     "/mediapipe/calculators/audio/"
-                     "testdata/sine_wave_1k_44100_mono_2_sec_wav.audio"));
+      file::JoinPath(GetTestDataDir(kTestPackageRoot),
+                     "sine_wave_1k_44100_mono_2_sec_wav.audio"));
   MP_ASSERT_OK(runner.Run());
   MP_EXPECT_OK(runner.Outputs()
                    .Tag("AUDIO_HEADER")
@@ -68,9 +71,8 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
         })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
-      file::JoinPath("./",
-                     "/mediapipe/calculators/audio/"
-                     "testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio"));
+      file::JoinPath(GetTestDataDir(kTestPackageRoot),
+                     "sine_wave_1k_48000_stereo_2_sec_wav.audio"));
   MP_ASSERT_OK(runner.Run());
   MP_EXPECT_OK(runner.Outputs()
                    .Tag("AUDIO_HEADER")
@@ -99,9 +101,8 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
         })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
-      file::JoinPath("./",
-                     "/mediapipe/calculators/audio/"
-                     "testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio"));
+      file::JoinPath(GetTestDataDir(kTestPackageRoot),
+                     "sine_wave_1k_44100_stereo_2_sec_mp3.audio"));
   MP_ASSERT_OK(runner.Run());
   MP_EXPECT_OK(runner.Outputs()
                    .Tag("AUDIO_HEADER")
@@ -130,9 +131,8 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
         })pb");
   CalculatorRunner runner(node_config);
   runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
-      file::JoinPath("./",
-                     "/mediapipe/calculators/audio/"
-                     "testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio"));
+      file::JoinPath(GetTestDataDir(kTestPackageRoot),
+                     "sine_wave_1k_44100_stereo_2_sec_aac.audio"));
   MP_ASSERT_OK(runner.Run());
   MP_EXPECT_OK(runner.Outputs()
                    .Tag("AUDIO_HEADER")
@@ -147,4 +147,5 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
               std::ceil(44100.0 * 2 / 1024));
 }
 
+}  // namespace
 }  // namespace mediapipe

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -20,24 +20,22 @@
 #include <memory>
 #include <string>
 
-#include "Eigen/Core"
 #include "absl/strings/string_view.h"
 #include "audio/dsp/spectrogram/spectrogram.h"
 #include "audio/dsp/window_functions.h"
 #include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
-#include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/core_proto_inc.h"
-#include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/source_location.h"
 #include "mediapipe/framework/port/status_builder.h"
 #include "mediapipe/util/time_series_util.h"
 
 namespace mediapipe {
 
+namespace {
+constexpr char kFrameDurationTag[] = "FRAME_DURATION";
+constexpr char kFrameOverlapTag[] = "FRAME_OVERLAP";
+}  // namespace
 // MediaPipe Calculator for computing the "spectrogram" (short-time Fourier
 // transform squared-magnitude, by default) of a multichannel input
 // time series, including optionally overlapping frames.  Options are
@@ -46,11 +44,14 @@ namespace mediapipe {
 //
 // Result is a MatrixData record (for single channel input and when the
 // allow_multichannel_input flag is false), or a vector of MatrixData records,
-// one for each channel (when the allow_multichannel_input flag is set). The
-// rows of each spectrogram matrix correspond to the n_fft/2+1 unique complex
-// values, or squared/linear/dB magnitudes, depending on the output_type option.
-// Each input packet will result in zero or one output packets, each containing
-// one Matrix for each channel of the input, where each Matrix has one or more
+// one for each channel (when the allow_multichannel_input flag is set). Each
+// waveform frame is converted to frequency by a fast Fourier transform whose
+// size, n_fft, is the smallest power of two large enough to enclose the frame
+// length of round(frame_duration_seconds * sample_rate).The rows of each
+// spectrogram matrix(result) correspond to the n_fft/2+1 unique complex values,
+// or squared/linear/dB magnitudes, depending on the output_type option. Each
+// input packet will result in zero or one output packets, each containing one
+// Matrix for each channel of the input, where each Matrix has one or more
 // columns of spectral values, one for each complete frame of input samples. If
 // the input packet contains too few samples to trigger a new output frame, no
 // output packet is generated (since zero-length packets are not legal since
@@ -71,6 +72,22 @@ class SpectrogramCalculator : public CalculatorBase {
         // Input stream with TimeSeriesHeader.
     );
 
+    if (cc->InputSidePackets().HasTag(kFrameDurationTag)) {
+      cc->InputSidePackets()
+          .Tag(kFrameDurationTag)
+          .Set<double>(
+              // Optional side packet for frame_duration_seconds if provided.
+          );
+    }
+
+    if (cc->InputSidePackets().HasTag(kFrameOverlapTag)) {
+      cc->InputSidePackets()
+          .Tag(kFrameOverlapTag)
+          .Set<double>(
+              // Optional side packet for frame_overlap_seconds if provided.
+          );
+    }
+
     SpectrogramCalculatorOptions spectrogram_options =
         cc->Options<SpectrogramCalculatorOptions>();
     if (!spectrogram_options.allow_multichannel_input()) {
@@ -184,27 +201,47 @@ class SpectrogramCalculator : public CalculatorBase {
   // Fixed scale factor applied to output values (regardless of type).
   double output_scale_;
 
-  static const float kLnPowerToDb;
+  static const float kLnSquaredMagnitudeToDb;
 };
 REGISTER_CALCULATOR(SpectrogramCalculator);
 
-// Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0).
-const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
+// DECIBELS = 20*log10(LINEAR_MAGNITUDE) = 10*Log10(SQUARED_MAGNITUDE)
+// =10/ln(10)*ln(SQUARED_MAGNITUDE).
+// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
+const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
 
 absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
   SpectrogramCalculatorOptions spectrogram_options =
       cc->Options<SpectrogramCalculatorOptions>();
+  // Provide frame_duration_seconds and frame_overlap_seconds either from static
+  // options, or dynamically from a side packet, the side packet one will
+  // override the options one if provided.
+
+  double frame_duration_seconds = 0;
+  double frame_overlap_seconds = 0;
+  if (cc->InputSidePackets().HasTag(kFrameDurationTag)) {
+    frame_duration_seconds =
+        cc->InputSidePackets().Tag(kFrameDurationTag).Get<double>();
+  } else {
+    frame_duration_seconds = spectrogram_options.frame_duration_seconds();
+  }
+
+  if (cc->InputSidePackets().HasTag(kFrameOverlapTag)) {
+    frame_overlap_seconds =
+        cc->InputSidePackets().Tag(kFrameOverlapTag).Get<double>();
+  } else {
+    frame_overlap_seconds = spectrogram_options.frame_overlap_seconds();
+  }
 
   use_local_timestamp_ = spectrogram_options.use_local_timestamp();
 
-  if (spectrogram_options.frame_duration_seconds() <= 0.0) {
+  if (frame_duration_seconds <= 0.0) {
     // TODO: return an error.
   }
-  if (spectrogram_options.frame_overlap_seconds() >=
-      spectrogram_options.frame_duration_seconds()) {
+  if (frame_overlap_seconds >= frame_duration_seconds) {
     // TODO: return an error.
   }
-  if (spectrogram_options.frame_overlap_seconds() < 0.0) {
+  if (frame_overlap_seconds < 0.0) {
     // TODO: return an error.
   }
 
@@ -220,10 +257,8 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
     // TODO: return an error.
   }
 
-  frame_duration_samples_ =
-      round(spectrogram_options.frame_duration_seconds() * input_sample_rate_);
-  frame_overlap_samples_ =
-      round(spectrogram_options.frame_overlap_seconds() * input_sample_rate_);
+  frame_duration_samples_ = round(frame_duration_seconds * input_sample_rate_);
+  frame_overlap_samples_ = round(frame_overlap_seconds * input_sample_rate_);
 
   pad_final_packet_ = spectrogram_options.pad_final_packet();
   output_type_ = spectrogram_options.output_type();
@@ -419,7 +454,7 @@ absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
       return ProcessVectorToOutput(
           input_stream,
           +[](const Matrix& col) -> const Matrix {
-            return kLnPowerToDb * col.array().log().matrix();
+            return kLnSquaredMagnitudeToDb * col.array().log().matrix();
           }, cc);
     }
     // clang-format on

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -32,7 +32,11 @@ message SpectrogramCalculatorOptions {
 
   // Duration of overlap between adjacent windows.
   // Hence, frame_rate = 1/(frame_duration_seconds - frame_overlap_seconds).
-  // Required that 0 <= frame_overlap_seconds <  frame_duration_seconds.
+  // Note the frame_rate here is not the MediaPipe packet rate, the frame here
+  // means each Fourier transform analysis waveform frame, the output MediaPipe
+  // packet rate will the the same as input, if frame rate is lower than input
+  // packet rate, will result in intermittent empty output packets. Required
+  // that 0 <= frame_overlap_seconds <  frame_duration_seconds.
   optional double frame_overlap_seconds = 2 [default = 0.0];
 
   // Whether to pad the final packet with zeros.  If true, guarantees that
@@ -42,6 +46,11 @@ message SpectrogramCalculatorOptions {
 
   // Output value type can be squared-magnitude, linear-magnitude,
   // deciBels (dB, = 20*log10(linear_magnitude)), or std::complex.
+  // Their relationship:
+  // COMPLEX c = Re + Im*i;
+  // SQUARED_MAGNITUDE = Re^2 + Im^2;
+  // LINEAR_MAGNITUDE = sqrt(SQUARED_MAGNITUDE);
+  // DECIBELS = 20*log10(LINEAR_MAGNITUDE) = 10*log10(SQUARED_MAGNITUDE);
   enum OutputType {
     SQUARED_MAGNITUDE = 0;
     LINEAR_MAGNITUDE = 1;

mediapipe/calculators/core/BUILD

@@ -117,6 +117,7 @@ mediapipe_proto_library(
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework/formats:classification_proto",
+        "//mediapipe/framework/formats:landmark_proto",
     ],
 )
 
@@ -213,6 +214,7 @@ cc_library(
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
@@ -309,8 +311,8 @@ cc_library(
 )
 
 cc_library(
-    name = "concatenate_normalized_landmark_list_calculator",
-    srcs = ["concatenate_normalized_landmark_list_calculator.cc"],
+    name = "concatenate_proto_list_calculator",
+    srcs = ["concatenate_proto_list_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",
@@ -324,10 +326,10 @@ cc_library(
 )
 
 cc_test(
-    name = "concatenate_normalized_landmark_list_calculator_test",
-    srcs = ["concatenate_normalized_landmark_list_calculator_test.cc"],
+    name = "concatenate_proto_list_calculator_test",
+    srcs = ["concatenate_proto_list_calculator_test.cc"],
     deps = [
-        ":concatenate_normalized_landmark_list_calculator",
+        ":concatenate_proto_list_calculator",
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
@@ -531,9 +533,13 @@ cc_test(
         ":split_vector_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/types:optional",
     ],
 )
 
@@ -551,6 +557,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "packet_cloner_calculator_test",
+    srcs = ["packet_cloner_calculator_test.cc"],
+    deps = [
+        ":packet_cloner_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "packet_inner_join_calculator",
     srcs = ["packet_inner_join_calculator.cc"],
@@ -960,8 +982,8 @@ cc_test(
 )
 
 cc_library(
-    name = "split_landmarks_calculator",
-    srcs = ["split_landmarks_calculator.cc"],
+    name = "split_proto_list_calculator",
+    srcs = ["split_proto_list_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
         ":split_vector_calculator_cc_proto",
@@ -975,10 +997,10 @@ cc_library(
 )
 
 cc_test(
-    name = "split_landmarks_calculator_test",
-    srcs = ["split_landmarks_calculator_test.cc"],
+    name = "split_proto_list_calculator_test",
+    srcs = ["split_proto_list_calculator_test.cc"],
     deps = [
-        ":split_landmarks_calculator",
+        ":split_proto_list_calculator",
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
@@ -1191,6 +1213,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -1238,7 +1261,6 @@ cc_test(
         "//mediapipe/framework:calculator_profile_cc_proto",
         "//mediapipe/framework:test_calculators",
         "//mediapipe/framework/deps:clock",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:core_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
@@ -1252,3 +1274,36 @@ cc_test(
         "@com_google_absl//absl/time",
     ],
 )
+
+cc_library(
+    name = "get_vector_item_calculator",
+    srcs = ["get_vector_item_calculator.cc"],
+    hdrs = ["get_vector_item_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "vector_size_calculator",
+    srcs = ["vector_size_calculator.cc"],
+    hdrs = ["vector_size_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -28,6 +28,10 @@ typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmarkList>>
     BeginLoopNormalizedLandmarkListVectorCalculator;
 REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator);
 
+// A calculator to process std::vector<int>.
+typedef BeginLoopCalculator<std::vector<int>> BeginLoopIntCalculator;
+REGISTER_CALCULATOR(BeginLoopIntCalculator);
+
 // A calculator to process std::vector<NormalizedRect>.
 typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
     BeginLoopNormalizedRectCalculator;
@@ -42,4 +46,13 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
 REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
 
+// A calculator to process std::vector<std::vector<Matrix>>.
+typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
+    BeginLoopMatrixVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixVectorCalculator);
+
+// A calculator to process std::vector<uint64_t>.
+typedef BeginLoopCalculator<std::vector<uint64_t>> BeginLoopUint64tCalculator;
+REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc

@@ -1,79 +0,0 @@
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_  // NOLINT
-#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_  // NOLINT
-
-#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
-#include "mediapipe/framework/api2/node.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/formats/landmark.pb.h"
-#include "mediapipe/framework/port/canonical_errors.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-
-namespace mediapipe {
-namespace api2 {
-
-// Concatenates several NormalizedLandmarkList protos following stream index
-// order. This class assumes that every input stream contains a
-// NormalizedLandmarkList proto object.
-class ConcatenateNormalizedLandmarkListCalculator : public Node {
- public:
-  static constexpr Input<NormalizedLandmarkList>::Multiple kIn{""};
-  static constexpr Output<NormalizedLandmarkList> kOut{""};
-
-  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
-
-  static absl::Status UpdateContract(CalculatorContract* cc) {
-    RET_CHECK_GE(kIn(cc).Count(), 1);
-    return absl::OkStatus();
-  }
-
-  absl::Status Open(CalculatorContext* cc) override {
-    only_emit_if_all_present_ =
-        cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
-            .only_emit_if_all_present();
-    return absl::OkStatus();
-  }
-
-  absl::Status Process(CalculatorContext* cc) override {
-    if (only_emit_if_all_present_) {
-      for (const auto& input : kIn(cc)) {
-        if (input.IsEmpty()) return absl::OkStatus();
-      }
-    }
-
-    NormalizedLandmarkList output;
-    for (const auto& input : kIn(cc)) {
-      if (input.IsEmpty()) continue;
-      const NormalizedLandmarkList& list = *input;
-      for (int j = 0; j < list.landmark_size(); ++j) {
-        *output.add_landmark() = list.landmark(j);
-      }
-    }
-    kOut(cc).Send(std::move(output));
-    return absl::OkStatus();
-  }
-
- private:
-  bool only_emit_if_all_present_;
-};
-MEDIAPIPE_REGISTER_NODE(ConcatenateNormalizedLandmarkListCalculator);
-
-}  // namespace api2
-}  // namespace mediapipe
-
-// NOLINTNEXTLINE
-#endif  // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_

mediapipe/calculators/core/concatenate_proto_list_calculator.cc

@@ -0,0 +1,118 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_PROTO_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_PROTO_LIST_CALCULATOR_H_  // NOLINT
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// Concatenate several input packets of ListType with a repeated field of
+// ItemType into a single output packet of ListType following stream index
+// order.
+template <typename ItemType, typename ListType>
+class ConcatenateListsCalculator : public Node {
+ public:
+  static constexpr typename Input<ListType>::Multiple kIn{""};
+  static constexpr Output<ListType> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_GE(kIn(cc).Count(), 1);
+    return absl::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    only_emit_if_all_present_ =
+        cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
+            .only_emit_if_all_present();
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (only_emit_if_all_present_) {
+      for (const auto& input : kIn(cc)) {
+        if (input.IsEmpty()) return absl::OkStatus();
+      }
+    }
+
+    ListType output;
+    for (const auto& input : kIn(cc)) {
+      if (input.IsEmpty()) continue;
+      const ListType& list = *input;
+      for (int j = 0; j < ListSize(list); ++j) {
+        *AddItem(output) = GetItem(list, j);
+      }
+    }
+    kOut(cc).Send(std::move(output));
+    return absl::OkStatus();
+  }
+
+ protected:
+  virtual int ListSize(const ListType& list) const = 0;
+  virtual const ItemType GetItem(const ListType& list, int idx) const = 0;
+  virtual ItemType* AddItem(ListType& list) const = 0;
+
+ private:
+  bool only_emit_if_all_present_;
+};
+
+// TODO: Move calculators to separate *.cc files
+
+class ConcatenateNormalizedLandmarkListCalculator
+    : public ConcatenateListsCalculator<NormalizedLandmark,
+                                        NormalizedLandmarkList> {
+ protected:
+  int ListSize(const NormalizedLandmarkList& list) const override {
+    return list.landmark_size();
+  }
+  const NormalizedLandmark GetItem(const NormalizedLandmarkList& list,
+                                   int idx) const override {
+    return list.landmark(idx);
+  }
+  NormalizedLandmark* AddItem(NormalizedLandmarkList& list) const override {
+    return list.add_landmark();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(ConcatenateNormalizedLandmarkListCalculator);
+
+class ConcatenateLandmarkListCalculator
+    : public ConcatenateListsCalculator<Landmark, LandmarkList> {
+ protected:
+  int ListSize(const LandmarkList& list) const override {
+    return list.landmark_size();
+  }
+  const Landmark GetItem(const LandmarkList& list, int idx) const override {
+    return list.landmark(idx);
+  }
+  Landmark* AddItem(LandmarkList& list) const override {
+    return list.add_landmark();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkListCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe
+
+// NOLINTNEXTLINE
+#endif  // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_PROTO_LIST_CALCULATOR_H_

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -73,8 +73,17 @@ typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
     ConcatenateLandmarkVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkVectorCalculator);
 
+typedef ConcatenateVectorCalculator<::mediapipe::LandmarkList>
+    ConcatenateLandmarkListVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkListVectorCalculator);
+
 typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmarkList>
-    ConcatenateLandmarListVectorCalculator;
+    ConcatenateNormalizedLandmarkListVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateNormalizedLandmarkListVectorCalculator);
+
+// For backwards compatibility, keep the version with the typo.
+using ConcatenateLandmarListVectorCalculator =
+    ConcatenateNormalizedLandmarkListVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarListVectorCalculator);
 
 typedef ConcatenateVectorCalculator<mediapipe::ClassificationList>

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -18,6 +18,7 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -79,6 +80,8 @@ class ConstantSidePacketCalculator : public CalculatorBase {
         packet.Set<uint64>();
       } else if (packet_options.has_classification_list_value()) {
         packet.Set<ClassificationList>();
+      } else if (packet_options.has_landmark_list_value()) {
+        packet.Set<LandmarkList>();
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
@@ -108,6 +111,9 @@ class ConstantSidePacketCalculator : public CalculatorBase {
       } else if (packet_options.has_classification_list_value()) {
         packet.Set(MakePacket<ClassificationList>(
             packet_options.classification_list_value()));
+      } else if (packet_options.has_landmark_list_value()) {
+        packet.Set(
+            MakePacket<LandmarkList>(packet_options.landmark_list_value()));
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -18,6 +18,7 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
+import "mediapipe/framework/formats/landmark.proto";
 
 option objc_class_prefix = "MediaPipe";
 
@@ -34,6 +35,7 @@ message ConstantSidePacketCalculatorOptions {
       string string_value = 4;
       uint64 uint64_value = 5;
       ClassificationList classification_list_value = 6;
+      LandmarkList landmark_list_value = 7;
     }
   }
 

mediapipe/calculators/core/end_loop_calculator.cc

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
@@ -50,4 +51,8 @@ REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
 typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
 REGISTER_CALCULATOR(EndLoopTensorCalculator);
 
+typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
+    EndLoopDetectionCalculator;
+REGISTER_CALCULATOR(EndLoopDetectionCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -32,8 +32,8 @@ constexpr char kOptionsTag[] = "OPTIONS";
 // FlowLimiterCalculator is used to limit the number of frames in flight
 // by dropping input frames when necessary.
 //
-// The input stream "FINISH" is used to signal the FlowLimiterCalculator
-// when a frame is finished processing.  Either a non-empty "FINISH" packet
+// The input stream "FINISHED" is used to signal the FlowLimiterCalculator
+// when a frame is finished processing.  Either a non-empty "FINISHED" packet
 // or a timestamp bound should be received for each processed frame.
 //
 // The combination of `max_in_flight: 1` and `max_in_queue: 1` generally gives

mediapipe/calculators/core/flow_limiter_calculator.proto

@@ -30,7 +30,7 @@ message FlowLimiterCalculatorOptions {
   optional int32 max_in_flight = 1 [default = 1];
 
   // The maximum number of frames queued waiting for processing.
-  // The default value limits to 1 frame awaiting processing.
+  // The default value limits to 0 frames awaiting processing.
   optional int32 max_in_queue = 2 [default = 0];
 
   // The maximum time in microseconds to wait for a frame to finish processing.

mediapipe/calculators/core/get_vector_item_calculator.cc

@@ -0,0 +1,32 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/get_vector_item_calculator.h"
+
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using GetLandmarkListVectorItemCalculator =
+    GetVectorItemCalculator<mediapipe::LandmarkList>;
+REGISTER_CALCULATOR(GetLandmarkListVectorItemCalculator);
+
+using GetClassificationListVectorItemCalculator =
+    GetVectorItemCalculator<mediapipe::ClassificationList>;
+REGISTER_CALCULATOR(GetClassificationListVectorItemCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/get_vector_item_calculator.h

@@ -0,0 +1,77 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_GET_VECTOR_ITEM_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_GET_VECTOR_ITEM_CALCULATOR_H_
+
+#include <optional>
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calcutlator to return an item from the vector by its index.
+//
+// Inputs:
+//   VECTOR - std::vector<T>
+//     Vector to take an item from.
+//   INDEX - int
+//     Index of the item to return.
+//
+// Outputs:
+//   ITEM - T
+//     Item from the vector at given index.
+//
+// Example config:
+//   node {
+//     calculator: "Get{SpecificType}VectorItemCalculator"
+//     input_stream: "VECTOR:vector"
+//     input_stream: "INDEX:index"
+//     input_stream: "ITEM:item"
+//   }
+//
+template <typename T>
+class GetVectorItemCalculator : public Node {
+ public:
+  static constexpr Input<std::vector<T>> kIn{"VECTOR"};
+  static constexpr Input<int> kIdx{"INDEX"};
+  static constexpr Output<T> kOut{"ITEM"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kIdx, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (kIn(cc).IsEmpty() || kIdx(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+
+    const std::vector<T>& items = kIn(cc).Get();
+    const int idx = kIdx(cc).Get();
+
+    RET_CHECK_LT(idx, items.size());
+    kOut(cc).Send(items[idx]);
+
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_GET_VECTOR_ITEM_CALCULATOR_H_

mediapipe/calculators/core/graph_profile_calculator.cc

@@ -29,6 +29,11 @@ namespace api2 {
 // This calculator periodically copies the GraphProfile from
 // mediapipe::GraphProfiler::CaptureProfile to the "PROFILE" output stream.
 //
+// Similarly to the log files saved by GraphProfiler::WriteProfile when trace
+// logging is enabled, the first captured profile contains the full
+// canonicalized graph config and, if tracing is enabled, calculator names in
+// graph traces. Subsequent profiles omit this information.
+//
 // Example config:
 // node {
 //   calculator: "GraphProfileCalculator"
@@ -50,11 +55,14 @@ class GraphProfileCalculator : public Node {
   absl::Status Process(CalculatorContext* cc) final {
     auto options = cc->Options<::mediapipe::GraphProfileCalculatorOptions>();
 
-    if (prev_profile_ts_ == Timestamp::Unset() ||
+    bool first_profile = prev_profile_ts_ == Timestamp::Unset();
+    if (first_profile ||
         cc->InputTimestamp() - prev_profile_ts_ >= options.profile_interval()) {
       prev_profile_ts_ = cc->InputTimestamp();
       GraphProfile result;
-      MP_RETURN_IF_ERROR(cc->GetProfilingContext()->CaptureProfile(&result));
+      MP_RETURN_IF_ERROR(cc->GetProfilingContext()->CaptureProfile(
+          &result, first_profile ? PopulateGraphConfig::kFull
+                                 : PopulateGraphConfig::kNo));
       kProfileOut(cc).Send(result);
     }
     return absl::OkStatus();

mediapipe/calculators/core/graph_profile_calculator_test.cc

@@ -23,7 +23,6 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_profile.pb.h"
 #include "mediapipe/framework/deps/clock.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -203,6 +202,8 @@ TEST_F(GraphProfileCalculatorTest, GraphProfile) {
           }
         })pb");
 
+  ASSERT_EQ(output_packets.size(), 2);
+  EXPECT_TRUE(output_packets[0].Get<GraphProfile>().has_config());
   EXPECT_THAT(output_packets[1].Get<GraphProfile>(),
               mediapipe::EqualsProto(expected_profile));
 }

mediapipe/calculators/core/mux_calculator_test.cc

@@ -14,7 +14,11 @@
 
 #include <memory>
 
+#include "absl/status/status.h"
+#include "absl/types/optional.h"
 #include "mediapipe/calculators/core/split_vector_calculator.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -301,4 +305,99 @@ TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
 }
 
 }  // namespace
+
+class PassThroughAndTsBoundUpdateNode : public mediapipe::api2::Node {
+ public:
+  static constexpr mediapipe::api2::Input<int> kInValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<int> kOutValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<int> kOutTsBoundUpdate{
+      "TS_BOUND_UPDATE"};
+  MEDIAPIPE_NODE_CONTRACT(kInValue, kOutValue, kOutTsBoundUpdate);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    kOutValue(cc).Send(kInValue(cc));
+    kOutTsBoundUpdate(cc).SetNextTimestampBound(
+        cc->InputTimestamp().NextAllowedInStream());
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(PassThroughAndTsBoundUpdateNode);
+
+class ToOptionalNode : public mediapipe::api2::Node {
+ public:
+  static constexpr mediapipe::api2::Input<int> kTick{"TICK"};
+  static constexpr mediapipe::api2::Input<int> kInValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<absl::optional<int>> kOutValue{
+      "OUTPUT"};
+  MEDIAPIPE_NODE_CONTRACT(kTick, kInValue, kOutValue);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (kInValue(cc).IsEmpty()) {
+      kOutValue(cc).Send(absl::nullopt);
+    } else {
+      kOutValue(cc).Send({kInValue(cc).Get()});
+    }
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(ToOptionalNode);
+
+namespace {
+
+TEST(MuxCalculatorTest, HandleTimestampBoundUpdates) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "select"
+            node {
+              calculator: "PassThroughAndTsBoundUpdateNode"
+              input_stream: "VALUE:select"
+              output_stream: "VALUE:select_ps"
+              output_stream: "TS_BOUND_UPDATE:ts_bound_update"
+            }
+            node {
+              calculator: "MuxCalculator"
+              input_stream: "INPUT:0:select_ps"
+              input_stream: "INPUT:1:ts_bound_update"
+              input_stream: "SELECT:select"
+              output_stream: "OUTPUT:select_or_ts_bound_update"
+            }
+            node {
+              calculator: "ToOptionalNode"
+              input_stream: "TICK:select"
+              input_stream: "VALUE:select_or_ts_bound_update"
+              output_stream: "OUTPUT:output"
+            }
+          )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto send_value_fn = [&](int value, Timestamp ts) -> absl::Status {
+    MP_RETURN_IF_ERROR(
+        graph.AddPacketToInputStream("select", MakePacket<int>(value).At(ts)));
+    return graph.WaitUntilIdle();
+  };
+
+  MP_ASSERT_OK(send_value_fn(0, Timestamp(1)));
+  ASSERT_EQ(output_packets.size(), 1);
+  EXPECT_EQ(output_packets[0].Get<absl::optional<int>>(), 0);
+
+  MP_ASSERT_OK(send_value_fn(1, Timestamp(2)));
+  ASSERT_EQ(output_packets.size(), 2);
+  EXPECT_EQ(output_packets[1].Get<absl::optional<int>>(), absl::nullopt);
+
+  MP_ASSERT_OK(send_value_fn(0, Timestamp(3)));
+  ASSERT_EQ(output_packets.size(), 3);
+  EXPECT_EQ(output_packets[2].Get<absl::optional<int>>(), 0);
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+}  // namespace
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_cloner_calculator.cc

@@ -16,9 +16,10 @@
 // For every packet that appears in B, outputs the most recent packet from each
 // of the A_i on a separate stream.
 
+#include <string_view>
 #include <vector>
 
-#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "mediapipe/calculators/core/packet_cloner_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 
@@ -34,7 +35,18 @@ namespace mediapipe {
 //   calculator: "PacketClonerCalculator"
 //   input_stream: "first_base_signal"
 //   input_stream: "second_base_signal"
-//   input_stream: "tick_signal"
+//   input_stream: "tick_signal"  # or input_stream: "TICK:tick_signal"
+//   output_stream: "cloned_first_base_signal"
+//   output_stream: "cloned_second_base_signal"
+// }
+//
+// Or you can use "TICK" tag and put corresponding input stream at any location,
+// for example at the very beginning:
+// node {
+//   calculator: "PacketClonerCalculator"
+//   input_stream: "TICK:tick_signal"
+//   input_stream: "first_base_signal"
+//   input_stream: "second_base_signal"
 //   output_stream: "cloned_first_base_signal"
 //   output_stream: "cloned_second_base_signal"
 // }
@@ -46,12 +58,13 @@ namespace mediapipe {
 class PacketClonerCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
-    const int tick_signal_index = cc->Inputs().NumEntries() - 1;
-    for (int i = 0; i < tick_signal_index; ++i) {
-      cc->Inputs().Index(i).SetAny();
-      cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
+    const Ids ids = GetIds(*cc);
+    for (const auto& in_out : ids.inputs_outputs) {
+      auto& input = cc->Inputs().Get(in_out.in);
+      input.SetAny();
+      cc->Outputs().Get(in_out.out).SetSameAs(&input);
     }
-    cc->Inputs().Index(tick_signal_index).SetAny();
+    cc->Inputs().Get(ids.tick_id).SetAny();
     return absl::OkStatus();
   }
 
@@ -60,15 +73,20 @@ class PacketClonerCalculator : public CalculatorBase {
     const auto calculator_options =
         cc->Options<mediapipe::PacketClonerCalculatorOptions>();
     output_only_when_all_inputs_received_ =
-        calculator_options.output_only_when_all_inputs_received();
+        calculator_options.output_only_when_all_inputs_received() ||
+        calculator_options.output_packets_only_when_all_inputs_received();
+    output_empty_packets_before_all_inputs_received_ =
+        calculator_options.output_packets_only_when_all_inputs_received();
+
+    // Prepare input and output ids.
+    ids_ = GetIds(*cc);
+    current_.resize(ids_.inputs_outputs.size());
 
-    // Parse input streams.
-    tick_signal_index_ = cc->Inputs().NumEntries() - 1;
-    current_.resize(tick_signal_index_);
     // Pass along the header for each stream if present.
-    for (int i = 0; i < tick_signal_index_; ++i) {
-      if (!cc->Inputs().Index(i).Header().IsEmpty()) {
-        cc->Outputs().Index(i).SetHeader(cc->Inputs().Index(i).Header());
+    for (const auto& in_out : ids_.inputs_outputs) {
+      auto& input = cc->Inputs().Get(in_out.in);
+      if (!input.Header().IsEmpty()) {
+        cc->Outputs().Get(in_out.out).SetHeader(input.Header());
       }
     }
     return absl::OkStatus();
@@ -76,29 +94,33 @@ class PacketClonerCalculator : public CalculatorBase {
 
   absl::Status Process(CalculatorContext* cc) final {
     // Store input signals.
-    for (int i = 0; i < tick_signal_index_; ++i) {
-      if (!cc->Inputs().Index(i).Value().IsEmpty()) {
-        current_[i] = cc->Inputs().Index(i).Value();
+    for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
+      const auto& input = cc->Inputs().Get(ids_.inputs_outputs[i].in);
+      if (!input.IsEmpty()) {
+        current_[i] = input.Value();
       }
     }
 
     // Output according to the TICK signal.
-    if (!cc->Inputs().Index(tick_signal_index_).Value().IsEmpty()) {
+    if (!cc->Inputs().Get(ids_.tick_id).IsEmpty()) {
       if (output_only_when_all_inputs_received_) {
         // Return if one of the input is null.
-        for (int i = 0; i < tick_signal_index_; ++i) {
+        for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
           if (current_[i].IsEmpty()) {
+            if (output_empty_packets_before_all_inputs_received_) {
+              SetAllNextTimestampBounds(cc);
+            }
             return absl::OkStatus();
           }
         }
       }
       // Output each stream.
-      for (int i = 0; i < tick_signal_index_; ++i) {
+      for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
+        auto& output = cc->Outputs().Get(ids_.inputs_outputs[i].out);
         if (!current_[i].IsEmpty()) {
-          cc->Outputs().Index(i).AddPacket(
-              current_[i].At(cc->InputTimestamp()));
+          output.AddPacket(current_[i].At(cc->InputTimestamp()));
         } else {
-          cc->Outputs().Index(i).SetNextTimestampBound(
+          output.SetNextTimestampBound(
               cc->InputTimestamp().NextAllowedInStream());
         }
       }
@@ -107,9 +129,46 @@ class PacketClonerCalculator : public CalculatorBase {
   }
 
  private:
+  struct Ids {
+    struct InputOutput {
+      CollectionItemId in;
+      CollectionItemId out;
+    };
+    CollectionItemId tick_id;
+    std::vector<InputOutput> inputs_outputs;
+  };
+
+  template <typename CC>
+  static Ids GetIds(CC& cc) {
+    Ids ids;
+    static constexpr absl::string_view kEmptyTag = "";
+    int num_inputs_to_clone = cc.Inputs().NumEntries(kEmptyTag);
+    static constexpr absl::string_view kTickTag = "TICK";
+    if (cc.Inputs().HasTag(kTickTag)) {
+      ids.tick_id = cc.Inputs().GetId(kTickTag, 0);
+    } else {
+      --num_inputs_to_clone;
+      ids.tick_id = cc.Inputs().GetId(kEmptyTag, num_inputs_to_clone);
+    }
+    for (int i = 0; i < num_inputs_to_clone; ++i) {
+      ids.inputs_outputs.push_back({.in = cc.Inputs().GetId(kEmptyTag, i),
+                                    .out = cc.Outputs().GetId(kEmptyTag, i)});
+    }
+    return ids;
+  }
+
+  void SetAllNextTimestampBounds(CalculatorContext* cc) {
+    for (const auto& in_out : ids_.inputs_outputs) {
+      cc->Outputs()
+          .Get(in_out.out)
+          .SetNextTimestampBound(cc->InputTimestamp().NextAllowedInStream());
+    }
+  }
+
   std::vector<Packet> current_;
-  int tick_signal_index_;
+  Ids ids_;
   bool output_only_when_all_inputs_received_;
+  bool output_empty_packets_before_all_inputs_received_;
 };
 
 REGISTER_CALCULATOR(PacketClonerCalculator);

mediapipe/calculators/core/packet_cloner_calculator.proto

@@ -28,4 +28,9 @@ message PacketClonerCalculatorOptions {
   // When true, this calculator will drop received TICK packets if any input
   // stream hasn't received a packet yet.
   optional bool output_only_when_all_inputs_received = 1 [default = false];
+
+  // Similar with above, but also transmit empty packet for all streams before
+  // all inputs are received.
+  optional bool output_packets_only_when_all_inputs_received = 2
+      [default = false];
 }

mediapipe/calculators/core/packet_cloner_calculator_test.cc

@@ -0,0 +1,349 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/sink.h"
+
+namespace mediapipe {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::Value;
+
+MATCHER_P2(IntPacket, value, ts, "") {
+  return Value(arg.template Get<int>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(Timestamp(ts)));
+}
+
+MATCHER_P2(FloatPacket, value, ts, "") {
+  return Value(arg.template Get<float>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(Timestamp(ts)));
+}
+
+template <typename T>
+absl::Status SendPacket(const std::string& input_name, T value, int ts,
+                        CalculatorGraph& graph) {
+  return graph.AddPacketToInputStream(input_name,
+                                      MakePacket<T>(value).At(Timestamp(ts)));
+}
+
+struct Params {
+  bool use_tick_tag = false;
+};
+
+class PacketClonerCalculatorTest : public testing::TestWithParam<Params> {};
+
+TEST_P(PacketClonerCalculatorTest, ClonesSingleInputSameTimestamps) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>([&]() {
+        if (GetParam().use_tick_tag) {
+          return R"pb(
+            input_stream: 'in1'
+            input_stream: 'tick'
+            node {
+              calculator: 'PacketClonerCalculator'
+              input_stream: 'in1'
+              input_stream: 'TICK:tick'
+              output_stream: 'out1'
+            })pb";
+        }
+        return R"pb(
+          input_stream: 'in1'
+          input_stream: 'tick'
+          node {
+            calculator: 'PacketClonerCalculator'
+            input_stream: 'in1'
+            input_stream: 'tick'
+            output_stream: 'out1'
+          })pb";
+      }());
+  std::vector<Packet> out1;
+  tool::AddVectorSink("out1", &graph_config, &out1);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  MP_ASSERT_OK(SendPacket("in1", 1, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1000, /*ts=*/10000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(out1, ElementsAre(IntPacket(1, 10000)));
+}
+
+TEST_P(PacketClonerCalculatorTest, ClonesSingleInputEarlierTimestamps) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>([&]() {
+        if (GetParam().use_tick_tag) {
+          return R"pb(
+            input_stream: 'in1'
+            input_stream: 'tick'
+            node {
+              calculator: 'PacketClonerCalculator'
+              input_stream: 'in1'
+              input_stream: 'TICK:tick'
+              output_stream: 'out1'
+            })pb";
+        }
+        return R"pb(
+          input_stream: 'in1'
+          input_stream: 'tick'
+          node {
+            calculator: 'PacketClonerCalculator'
+            input_stream: 'in1'
+            input_stream: 'tick'
+            output_stream: 'out1'
+          })pb";
+      }());
+  std::vector<Packet> out1;
+  tool::AddVectorSink("out1", &graph_config, &out1);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // PacketClonerCalculator is non-ImmediateInputStreamHandler
+  // PacketClonerCalculator waits for "in1" to arrive for ts=5000
+  MP_ASSERT_OK(SendPacket("in1", 1, /*ts=*/5000, graph));
+  // Newer tick at ts=10000, should NOT trigger output for ts=5000
+  // PacketClonerCalculator waits for "in1" to arrive for ts=10000
+  MP_ASSERT_OK(SendPacket("tick", 1000, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1001, /*ts=*/10001, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1002, /*ts=*/10002, graph));
+  // Newer "in1" at ts=15000, should trigger output for ts=10000
+  MP_ASSERT_OK(SendPacket("in1", 2, /*ts=*/15000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(out1, ElementsAre(IntPacket(1, 10000), IntPacket(1, 10001),
+                                IntPacket(1, 10002)));
+}
+
+TEST_P(PacketClonerCalculatorTest, ClonesFiveInputs) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>([&]() {
+        if (GetParam().use_tick_tag) {
+          return R"pb(
+            input_stream: 'in1'
+            input_stream: 'in2'
+            input_stream: 'in3'
+            input_stream: 'in4'
+            input_stream: 'in5'
+            input_stream: 'tick'
+            node {
+              calculator: 'PacketClonerCalculator'
+              input_stream: 'in1'
+              input_stream: 'in2'
+              input_stream: 'in3'
+              input_stream: 'in4'
+              input_stream: 'in5'
+              output_stream: 'out1'
+              output_stream: 'out2'
+              output_stream: 'out3'
+              input_stream: 'TICK:tick'  # arbitrary location
+              output_stream: 'out4'
+              output_stream: 'out5'
+            }
+          )pb";
+        }
+        return R"pb(
+          input_stream: 'in1'
+          input_stream: 'in2'
+          input_stream: 'in3'
+          input_stream: 'in4'
+          input_stream: 'in5'
+          input_stream: 'tick'
+          node {
+            calculator: 'PacketClonerCalculator'
+            input_stream: 'in1'
+            input_stream: 'in2'
+            input_stream: 'in3'
+            input_stream: 'in4'
+            input_stream: 'in5'
+            input_stream: 'tick'
+            output_stream: 'out1'
+            output_stream: 'out2'
+            output_stream: 'out3'
+            output_stream: 'out4'
+            output_stream: 'out5'
+          }
+        )pb";
+      }());
+  constexpr int kNumToClone = 5;
+  std::array<std::vector<Packet>, kNumToClone> outs;
+  for (int i = 0; i < kNumToClone; ++i) {
+    tool::AddVectorSink(absl::StrCat("out", i + 1), &graph_config, &outs[i]);
+  }
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  MP_ASSERT_OK(SendPacket("in1", 10, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("in2", 20.0f, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("in3", 30, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("in4", 40.0f, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("in5", 50, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1000, /*ts=*/10000, graph));
+  // Below "tick" packets won't trigger output, until newer inputs are sent,
+  // because inputs are missing and ImmediateInputStreamHandler is not
+  // configured.
+  MP_ASSERT_OK(SendPacket("tick", 1001, /*ts=*/10001, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1002, /*ts=*/10002, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(outs, ElementsAre(ElementsAre(IntPacket(10, 10000)),
+                                ElementsAre(FloatPacket(20.0f, 10000)),
+                                ElementsAre(IntPacket(30, 10000)),
+                                ElementsAre(FloatPacket(40.0f, 10000)),
+                                ElementsAre(IntPacket(50, 10000))));
+
+  MP_ASSERT_OK(SendPacket("in1", 100, /*ts=*/20000, graph));
+  MP_ASSERT_OK(SendPacket("in2", 200.0f, /*ts=*/20000, graph));
+  MP_ASSERT_OK(SendPacket("in3", 300, /*ts=*/20000, graph));
+  MP_ASSERT_OK(SendPacket("in4", 400.0f, /*ts=*/20000, graph));
+  MP_ASSERT_OK(SendPacket("in5", 500, /*ts=*/20000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 2000, /*ts=*/20000, graph));
+  // Below "tick" packets won't trigger output, because inputs are missing and
+  // ImmediateInputStreamHandler is not configured.
+  MP_ASSERT_OK(SendPacket("tick", 2001, /*ts=*/20001, graph));
+  MP_ASSERT_OK(SendPacket("tick", 2002, /*ts=*/20002, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(
+      outs,
+      ElementsAre(
+          ElementsAre(IntPacket(10, 10000), IntPacket(10, 10001),
+                      IntPacket(10, 10002), IntPacket(100, 20000)),
+          ElementsAre(FloatPacket(20.0f, 10000), FloatPacket(20.0f, 10001),
+                      FloatPacket(20.0f, 10002), FloatPacket(200.0f, 20000)),
+          ElementsAre(IntPacket(30, 10000), IntPacket(30, 10001),
+                      IntPacket(30, 10002), IntPacket(300, 20000)),
+          ElementsAre(FloatPacket(40.0f, 10000), FloatPacket(40.0f, 10001),
+                      FloatPacket(40.0f, 10002), FloatPacket(400.0f, 20000)),
+          ElementsAre(IntPacket(50, 10000), IntPacket(50, 10001),
+                      IntPacket(50, 10002), IntPacket(500, 20000))));
+}
+
+TEST_P(PacketClonerCalculatorTest,
+       ClonesTwoInputsWithImmediateInputStreamHandler) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>([&]() {
+        if (GetParam().use_tick_tag) {
+          return R"pb(
+            input_stream: 'in1'
+            input_stream: 'in2'
+            input_stream: 'tick'
+            node {
+              calculator: 'PacketClonerCalculator'
+              input_stream: 'TICK:tick'
+              input_stream: 'in1'
+              input_stream: 'in2'
+              output_stream: 'out1'
+              output_stream: 'out2'
+              input_stream_handler {
+                input_stream_handler: "ImmediateInputStreamHandler"
+              }
+            })pb";
+        }
+        return R"pb(
+          input_stream: 'in1'
+          input_stream: 'in2'
+          input_stream: 'tick'
+          node {
+            calculator: 'PacketClonerCalculator'
+            input_stream: 'in1'
+            input_stream: 'in2'
+            input_stream: 'tick'
+            output_stream: 'out1'
+            output_stream: 'out2'
+            input_stream_handler {
+              input_stream_handler: "ImmediateInputStreamHandler"
+            }
+          })pb";
+      }());
+  constexpr int kNumToClone = 2;
+  std::array<std::vector<Packet>, kNumToClone> outs;
+  for (int i = 0; i < kNumToClone; ++i) {
+    tool::AddVectorSink(absl::StrCat("out", i + 1), &graph_config, &outs[i]);
+  }
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // No packets to clone.
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/0, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Cloning current packets.
+  MP_ASSERT_OK(SendPacket("in1", 1, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("in2", 10.0f, /*ts=*/10000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 1000, /*ts=*/10000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Cloning past packets.
+  MP_ASSERT_OK(SendPacket("tick", 1500, /*ts=*/15000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Cloning past packets.
+  MP_ASSERT_OK(SendPacket("in1", 2, /*ts=*/10001, graph));
+  MP_ASSERT_OK(SendPacket("in2", 20.0f, /*ts=*/10001, graph));
+  MP_ASSERT_OK(SendPacket("tick", 2000, /*ts=*/20000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Cloning future packets.
+  MP_ASSERT_OK(SendPacket("in1", 3, /*ts=*/30000, graph));
+  MP_ASSERT_OK(SendPacket("in2", 30.0f, /*ts=*/30000, graph));
+  // Waiting to ensure newer packets (ts=30000) to clone would get into the
+  // cloner before tick (ts=25000) does.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(SendPacket("tick", 3000, /*ts=*/25000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Cloning packets having different timestamps.
+  MP_ASSERT_OK(SendPacket("in1", 4, /*ts=*/38000, graph));
+  MP_ASSERT_OK(SendPacket("in2", 40.0f, /*ts=*/39000, graph));
+  MP_ASSERT_OK(SendPacket("tick", 4000, /*ts=*/40000, graph));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(
+      outs,
+      ElementsAre(
+          ElementsAre(IntPacket(1, 10000), IntPacket(1, 15000),
+                      IntPacket(2, 20000), IntPacket(3, 25000),
+                      IntPacket(4, 40000)),
+          ElementsAre(FloatPacket(10.0f, 10000), FloatPacket(10.0f, 15000),
+                      FloatPacket(20.0f, 20000), FloatPacket(30.0f, 25000),
+                      FloatPacket(40.0f, 40000))));
+}
+
+INSTANTIATE_TEST_SUITE_P(PacketClonerCalculator, PacketClonerCalculatorTest,
+                         testing::ValuesIn({Params{.use_tick_tag = false},
+                                            Params{.use_tick_tag = true}}));
+}  // anonymous namespace
+}  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -157,9 +157,7 @@ absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) {
     }
   }
 
-  if (absl::Status status = strategy_->Process(cc); !status.ok()) {
-    return status;  // Avoid MP_RETURN_IF_ERROR macro for external release.
-  }
+  MP_RETURN_IF_ERROR(strategy_->Process(cc));
 
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
 

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -217,6 +217,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
       header->format = video_header.format;
       header->width = video_header.width;
       header->height = video_header.height;
+      header->duration = video_header.duration;
       header->frame_rate = new_frame_rate;
       cc->Outputs().Index(0).SetHeader(Adopt(header.release()));
     } else {

mediapipe/calculators/core/quantize_float_vector_calculator.cc

@@ -23,8 +23,8 @@
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/status.h"
 
-// Quantizes a vector of floats to a std::string so that each float becomes a
-// byte in the [0, 255] range. Any value above max_quantized_value or below
+// Quantizes a vector of floats to a string so that each float becomes a byte
+// in the [0, 255] range. Any value above max_quantized_value or below
 // min_quantized_value will be saturated to '/xFF' or '/0'.
 //
 // Example config:

mediapipe/calculators/core/split_proto_list_calculator.cc

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_  // NOLINT
-#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_  // NOLINT
+#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_PROTO_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_PROTO_LIST_CALCULATOR_H_  // NOLINT
 
 #include "mediapipe/calculators/core/split_vector_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -24,30 +24,30 @@
 
 namespace mediapipe {
 
-// Splits an input packet with LandmarkListType into
-// multiple LandmarkListType output packets using the [begin, end) ranges
+// Splits an input packet of ListType with a repeated field of ItemType
+// into multiple ListType output packets using the [begin, end) ranges
 // specified in SplitVectorCalculatorOptions. If the option "element_only" is
 // set to true, all ranges should be of size 1 and all outputs will be elements
-// of type LandmarkType. If "element_only" is false, ranges can be
-// non-zero in size and all outputs will be of type LandmarkListType.
+// of type ItemType. If "element_only" is false, ranges can be
+// non-zero in size and all outputs will be of type ListType.
 // If the option "combine_outputs" is set to true, only one output stream can be
 // specified and all ranges of elements will be combined into one
-// LandmarkListType.
-template <typename LandmarkType, typename LandmarkListType>
-class SplitLandmarksCalculator : public CalculatorBase {
+// ListType.
+template <typename ItemType, typename ListType>
+class SplitListsCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
     RET_CHECK(cc->Inputs().NumEntries() == 1);
     RET_CHECK(cc->Outputs().NumEntries() != 0);
 
-    cc->Inputs().Index(0).Set<LandmarkListType>();
+    cc->Inputs().Index(0).Set<ListType>();
 
     const auto& options =
         cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
 
     if (options.combine_outputs()) {
       RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
-      cc->Outputs().Index(0).Set<LandmarkListType>();
+      cc->Outputs().Index(0).Set<ListType>();
       for (int i = 0; i < options.ranges_size() - 1; ++i) {
         for (int j = i + 1; j < options.ranges_size(); ++j) {
           const auto& range_0 = options.ranges(i);
@@ -82,9 +82,9 @@ class SplitLandmarksCalculator : public CalculatorBase {
             return absl::InvalidArgumentError(
                 "Since element_only is true, all ranges should be of size 1.");
           }
-          cc->Outputs().Index(i).Set<LandmarkType>();
+          cc->Outputs().Index(i).Set<ItemType>();
         } else {
-          cc->Outputs().Index(i).Set<LandmarkListType>();
+          cc->Outputs().Index(i).Set<ListType>();
         }
       }
     }
@@ -111,39 +111,38 @@ class SplitLandmarksCalculator : public CalculatorBase {
   }
 
   absl::Status Process(CalculatorContext* cc) override {
-    const LandmarkListType& input =
-        cc->Inputs().Index(0).Get<LandmarkListType>();
-    RET_CHECK_GE(input.landmark_size(), max_range_end_)
-        << "Max range end " << max_range_end_ << " exceeds landmarks size "
-        << input.landmark_size();
+    const ListType& input = cc->Inputs().Index(0).Get<ListType>();
+    RET_CHECK_GE(ListSize(input), max_range_end_)
+        << "Max range end " << max_range_end_ << " exceeds list size "
+        << ListSize(input);
 
     if (combine_outputs_) {
-      LandmarkListType output;
+      ListType output;
       for (int i = 0; i < ranges_.size(); ++i) {
         for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
-          const LandmarkType& input_landmark = input.landmark(j);
-          *output.add_landmark() = input_landmark;
+          const ItemType& input_item = GetItem(input, j);
+          *AddItem(output) = input_item;
         }
       }
-      RET_CHECK_EQ(output.landmark_size(), total_elements_);
+      RET_CHECK_EQ(ListSize(output), total_elements_);
       cc->Outputs().Index(0).AddPacket(
-          MakePacket<LandmarkListType>(output).At(cc->InputTimestamp()));
+          MakePacket<ListType>(output).At(cc->InputTimestamp()));
     } else {
       if (element_only_) {
         for (int i = 0; i < ranges_.size(); ++i) {
           cc->Outputs().Index(i).AddPacket(
-              MakePacket<LandmarkType>(input.landmark(ranges_[i].first))
+              MakePacket<ItemType>(GetItem(input, ranges_[i].first))
                   .At(cc->InputTimestamp()));
         }
       } else {
         for (int i = 0; i < ranges_.size(); ++i) {
-          LandmarkListType output;
+          ListType output;
           for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
-            const LandmarkType& input_landmark = input.landmark(j);
-            *output.add_landmark() = input_landmark;
+            const ItemType& input_item = GetItem(input, j);
+            *AddItem(output) = input_item;
           }
           cc->Outputs().Index(i).AddPacket(
-              MakePacket<LandmarkListType>(output).At(cc->InputTimestamp()));
+              MakePacket<ListType>(output).At(cc->InputTimestamp()));
         }
       }
     }
@@ -151,6 +150,11 @@ class SplitLandmarksCalculator : public CalculatorBase {
     return absl::OkStatus();
   }
 
+ protected:
+  virtual int ListSize(const ListType& list) const = 0;
+  virtual const ItemType GetItem(const ListType& list, int idx) const = 0;
+  virtual ItemType* AddItem(ListType& list) const = 0;
+
  private:
   std::vector<std::pair<int32, int32>> ranges_;
   int32 max_range_end_ = -1;
@@ -159,15 +163,40 @@ class SplitLandmarksCalculator : public CalculatorBase {
   bool combine_outputs_ = false;
 };
 
-typedef SplitLandmarksCalculator<NormalizedLandmark, NormalizedLandmarkList>
-    SplitNormalizedLandmarkListCalculator;
+// TODO: Move calculators to separate *.cc files
+
+class SplitNormalizedLandmarkListCalculator
+    : public SplitListsCalculator<NormalizedLandmark, NormalizedLandmarkList> {
+ protected:
+  int ListSize(const NormalizedLandmarkList& list) const override {
+    return list.landmark_size();
+  }
+  const NormalizedLandmark GetItem(const NormalizedLandmarkList& list,
+                                   int idx) const override {
+    return list.landmark(idx);
+  }
+  NormalizedLandmark* AddItem(NormalizedLandmarkList& list) const override {
+    return list.add_landmark();
+  }
+};
 REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator);
 
-typedef SplitLandmarksCalculator<Landmark, LandmarkList>
-    SplitLandmarkListCalculator;
+class SplitLandmarkListCalculator
+    : public SplitListsCalculator<Landmark, LandmarkList> {
+ protected:
+  int ListSize(const LandmarkList& list) const override {
+    return list.landmark_size();
+  }
+  const Landmark GetItem(const LandmarkList& list, int idx) const override {
+    return list.landmark(idx);
+  }
+  Landmark* AddItem(LandmarkList& list) const override {
+    return list.add_landmark();
+  }
+};
 REGISTER_CALCULATOR(SplitLandmarkListCalculator);
 
 }  // namespace mediapipe
 
 // NOLINTNEXTLINE
-#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_LANDMARKS_CALCULATOR_H_
+#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_PROTO_LIST_CALCULATOR_H_

mediapipe/calculators/core/split_vector_calculator.cc

@@ -80,4 +80,10 @@ typedef SplitVectorCalculator<mediapipe::ClassificationList, false>
     SplitClassificationListVectorCalculator;
 REGISTER_CALCULATOR(SplitClassificationListVectorCalculator);
 
+typedef SplitVectorCalculator<uint64_t, false> SplitUint64tVectorCalculator;
+REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
+
+typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
+REGISTER_CALCULATOR(SplitFloatVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/string_to_int_calculator.cc

@@ -24,7 +24,7 @@
 
 namespace mediapipe {
 
-// Calculator that converts a std::string into an integer type, or fails if the
+// Calculator that converts a string into an integer type, or fails if the
 // conversion is not possible.
 //
 // Example config:
@@ -47,7 +47,7 @@ class StringToIntCalculatorTemplate : public CalculatorBase {
     if (!absl::SimpleAtoi(cc->InputSidePackets().Index(0).Get<std::string>(),
                           &number)) {
       return absl::InvalidArgumentError(
-          "The std::string could not be parsed as an integer.");
+          "The string could not be parsed as an integer.");
     }
     cc->OutputSidePackets().Index(0).Set(MakePacket<IntType>(number));
     return absl::OkStatus();

mediapipe/calculators/core/vector_size_calculator.cc

@@ -0,0 +1,32 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/vector_size_calculator.h"
+
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using LandmarkListVectorSizeCalculator =
+    VectorSizeCalculator<mediapipe::LandmarkList>;
+REGISTER_CALCULATOR(LandmarkListVectorSizeCalculator);
+
+using ClassificationListVectorSizeCalculator =
+    VectorSizeCalculator<mediapipe::ClassificationList>;
+REGISTER_CALCULATOR(ClassificationListVectorSizeCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/vector_size_calculator.h

@@ -0,0 +1,64 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_VECTOR_SIZE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_VECTOR_SIZE_CALCULATOR_H_
+
+#include <optional>
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calcutlator to return vector size.
+//
+// Inputs:
+//   VECTOR - std::vector<T>
+//     Vector which size to return.
+//
+// Outputs:
+//   SIZE - int
+//     Size of the input vector.
+//
+// Example config:
+//  node {
+//    calculator: "{SpecificType}VectorSizeCalculator"
+//    input_stream: "VECTOR:vector"
+//    output_stream: "SIZE:vector_size"
+//  }
+//
+template <typename T>
+class VectorSizeCalculator : public Node {
+ public:
+  static constexpr Input<std::vector<T>> kIn{"VECTOR"};
+  static constexpr Output<int> kOut{"SIZE"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (kIn(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+    kOut(cc).Send(kIn(cc).Get().size());
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_VECTOR_SIZE_CALCULATOR_H_

mediapipe/calculators/image/BUILD

@@ -239,10 +239,13 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":image_transformation_calculator_cc_proto",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
         "//mediapipe/gpu:scale_mode_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
@@ -623,11 +626,8 @@ cc_library(
         "//mediapipe/framework/formats:image_format_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
-        "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:image",
-        "//mediapipe/framework/formats:image_opencv",
         "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ] + select({
@@ -638,6 +638,13 @@ cc_library(
             "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
+    }) + select({
+        "//mediapipe/framework/port:disable_opencv": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:image_frame_opencv",
+            "//mediapipe/framework/formats:image_opencv",
+            "//mediapipe/framework/port:opencv_core",
+        ],
     }),
     alwayslink = 1,
 )
@@ -724,7 +731,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":affine_transformation",
-        ":affine_transformation_runner_opencv",
         ":warp_affine_calculator_cc_proto",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
@@ -742,6 +748,9 @@ cc_library(
             "//mediapipe/gpu:gpu_buffer",
             ":affine_transformation_runner_gl",
         ],
+    }) + select({
+        "//mediapipe/framework/port:disable_opencv": [],
+        "//conditions:default": [":affine_transformation_runner_opencv"],
     }),
     alwayslink = 1,
 )
@@ -796,3 +805,21 @@ cc_test(
         "@com_google_absl//absl/strings",
     ],
 )
+
+cc_library(
+    name = "yuv_to_image_calculator",
+    srcs = ["yuv_to_image_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:yuv_image",
+        "//third_party/libyuv",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:str_format",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -480,8 +480,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
   if (cc->Inputs().HasTag(kRectTag)) {
     const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     // Only use the rect if it is valid.
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
+    if (rect.width() > 0 && rect.height() > 0) {
       x_center = rect.x_center();
       y_center = rect.y_center();
       crop_width = rect.width();

mediapipe/calculators/image/image_file_properties_calculator.cc

@@ -105,7 +105,7 @@ absl::StatusOr<ImageFileProperties> GetImageFileProperites(
 }  // namespace
 
 // Calculator to extract EXIF information from an image file. The input is
-// a std::string containing raw byte data from a file, and the output is an
+// a string containing raw byte data from a file, and the output is an
 // ImageFileProperties proto object with the relevant fields filled in.
 // The calculator accepts the input as a stream or a side packet, and can output
 // the result as a stream or a side packet. The calculator checks that if an

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -16,10 +16,13 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/gpu/scale_mode.pb.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
@@ -45,10 +48,14 @@ namespace mediapipe {
 #if !MEDIAPIPE_DISABLE_GPU
 
 #endif  // !MEDIAPIPE_DISABLE_GPU
+#if defined(MEDIAPIPE_IOS)
+
+#endif  // defined(MEDIAPIPE_IOS)
 
 namespace {
 constexpr char kImageFrameTag[] = "IMAGE";
 constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
 
 int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
   switch (rotation) {
@@ -119,6 +126,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   provided, it overrides the FLIP_VERTICALLY input side packet and/or
 //   corresponding field in the calculator options.
 //
+//   VIDEO_PRESTREAM (optional): VideoHeader for the input ImageFrames, if
+//   rotating or scaling the frames, the header width and height will be updated
+//   appropriately. Note the header is updated only based on dimensions and
+//   rotations specified as side packets or options, input_stream
+//   transformations will not update the header.
+//
 // Output:
 //   One of the following tags:
 //   IMAGE - ImageFrame representing the output image.
@@ -239,6 +252,21 @@ absl::Status ImageTransformationCalculator::GetContract(
     cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
   }
 
+  RET_CHECK(cc->Inputs().HasTag(kVideoPrestreamTag) ==
+            cc->Outputs().HasTag(kVideoPrestreamTag))
+      << "If VIDEO_PRESTREAM is provided, it must be provided both as an "
+         "inputs and output stream.";
+  if (cc->Inputs().HasTag(kVideoPrestreamTag)) {
+    RET_CHECK(!(cc->Inputs().HasTag("OUTPUT_DIMENSIONS") ||
+                cc->Inputs().HasTag("ROTATION_DEGREES")))
+        << "If specifying VIDEO_PRESTREAM, the transformations that affect the "
+           "dimensions of the frames (OUTPUT_DIMENSIONS and ROTATION_DEGREES) "
+           "need to be constant for every frame, meaning they can only be "
+           "provided in the calculator options or side packets.";
+    cc->Inputs().Tag(kVideoPrestreamTag).Set<mediapipe::VideoHeader>();
+    cc->Outputs().Tag(kVideoPrestreamTag).Set<mediapipe::VideoHeader>();
+  }
+
   if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
     cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
   }
@@ -323,6 +351,24 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
 }
 
 absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
+  // First update the video header if it is given, based on the rotation and
+  // dimensions specified as side packets or options. This will only be done
+  // once, so streaming transformation changes will not be reflected in
+  // the header.
+  if (cc->Inputs().HasTag(kVideoPrestreamTag) &&
+      !cc->Inputs().Tag(kVideoPrestreamTag).IsEmpty() &&
+      cc->Outputs().HasTag(kVideoPrestreamTag)) {
+    mediapipe::VideoHeader header =
+        cc->Inputs().Tag(kVideoPrestreamTag).Get<mediapipe::VideoHeader>();
+    // Update the header's width and height if needed.
+    ComputeOutputDimensions(header.width, header.height, &header.width,
+                            &header.height);
+    cc->Outputs()
+        .Tag(kVideoPrestreamTag)
+        .AddPacket(mediapipe::MakePacket<mediapipe::VideoHeader>(header).At(
+            mediapipe::Timestamp::PreStream()));
+  }
+
   // Override values if specified so.
   if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
       !cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
@@ -337,12 +383,15 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
       !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
     flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
   }
-  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS") &&
-      !cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
-    const auto& image_size =
-        cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
-    output_width_ = image_size.first;
-    output_height_ = image_size.second;
+  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) {
+    if (cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
+      return absl::OkStatus();
+    } else {
+      const auto& image_size =
+          cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
+      output_width_ = image_size.first;
+      output_height_ = image_size.second;
+    }
   }
 
   if (use_gpu_) {
@@ -506,6 +555,14 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
   ComputeOutputDimensions(input_width, input_height, &output_width,
                           &output_height);
 
+  if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
+    const float scale =
+        std::min(static_cast<float>(output_width_) / input_width,
+                 static_cast<float>(output_height_) / input_height);
+    output_width = std::round(input_width * scale);
+    output_height = std::round(input_height * scale);
+  }
+
   if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
     auto padding = absl::make_unique<std::array<float, 4>>();
     ComputeOutputLetterboxPadding(input_width, input_height, output_width,

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc

@@ -22,9 +22,9 @@
 
 namespace mediapipe {
 
-// Takes in an encoded image std::string, decodes it by OpenCV, and converts to
-// an ImageFrame. Note that this calculator only supports grayscale and RGB
-// images for now.
+// Takes in an encoded image string, decodes it by OpenCV, and converts to an
+// ImageFrame. Note that this calculator only supports grayscale and RGB images
+// for now.
 //
 // Example config:
 // node {

mediapipe/calculators/image/opencv_put_text_calculator.cc

@@ -20,8 +20,8 @@
 
 namespace mediapipe {
 
-// Takes in a std::string, draws the text std::string by cv::putText(), and
-// outputs an ImageFrame.
+// Takes in a string, draws the text string by cv::putText(), and outputs an
+// ImageFrame.
 //
 // Example config:
 // node {

mediapipe/calculators/image/scale_image_calculator.cc

@@ -421,6 +421,10 @@ absl::Status ScaleImageCalculator::InitializeFromOptions() {
     alignment_boundary_ = options_.alignment_boundary();
   }
 
+  if (options_.has_output_format()) {
+    output_format_ = options_.output_format();
+  }
+
   downscaler_.reset(new ImageResizer(options_.post_sharpening_coefficient()));
 
   return absl::OkStatus();
@@ -433,13 +437,17 @@ absl::Status ScaleImageCalculator::ValidateImageFormats() const {
       << "The output image format was set to UNKNOWN.";
   // TODO Remove these conditions.
   RET_CHECK(output_format_ == ImageFormat::SRGB ||
+            output_format_ == ImageFormat::SRGBA ||
             (input_format_ == output_format_ &&
              output_format_ == ImageFormat::YCBCR420P))
       << "Outputting YCbCr420P images from SRGB input is not yet supported";
   RET_CHECK(input_format_ == output_format_ ||
-            input_format_ == ImageFormat::YCBCR420P)
+            (input_format_ == ImageFormat::YCBCR420P &&
+             output_format_ == ImageFormat::SRGB) ||
+            (input_format_ == ImageFormat::SRGB &&
+             output_format_ == ImageFormat::SRGBA))
       << "Conversion of the color space (except from "
-         "YCbCr420P to SRGB) is not yet supported.";
+         "YCbCr420P to SRGB or SRGB to SRBGA) is not yet supported.";
   return absl::OkStatus();
 }
 
@@ -553,7 +561,6 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
     }
   }
 
-  cc->GetCounter("Inputs")->Increment();
   const ImageFrame* image_frame;
   ImageFrame converted_image_frame;
   if (input_format_ == ImageFormat::YCBCR420P) {
@@ -605,6 +612,15 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
           .Add(output_image.release(), cc->InputTimestamp());
       return absl::OkStatus();
     }
+  } else if (input_format_ == ImageFormat::SRGB &&
+             output_format_ == ImageFormat::SRGBA) {
+    image_frame = &cc->Inputs().Get(input_data_id_).Get<ImageFrame>();
+    cv::Mat input_mat = ::mediapipe::formats::MatView(image_frame);
+    converted_image_frame.Reset(ImageFormat::SRGBA, image_frame->Width(),
+                                image_frame->Height(), alignment_boundary_);
+    cv::Mat output_mat = ::mediapipe::formats::MatView(&converted_image_frame);
+    cv::cvtColor(input_mat, output_mat, cv::COLOR_RGB2RGBA, 4);
+    image_frame = &converted_image_frame;
   } else {
     image_frame = &cc->Inputs().Get(input_data_id_).Get<ImageFrame>();
     MP_RETURN_IF_ERROR(ValidateImageFrame(cc, *image_frame));

mediapipe/calculators/image/segmentation_smoothing_calculator.cc

@@ -21,10 +21,7 @@
 #include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/framework/formats/image_frame_opencv.h"
-#include "mediapipe/framework/formats/image_opencv.h"
 #include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 
@@ -34,6 +31,12 @@
 #include "mediapipe/gpu/shader_util.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
+#if !MEDIAPIPE_DISABLE_OPENCV
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/image_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
+
 namespace mediapipe {
 
 namespace {
@@ -163,7 +166,11 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) {
     return absl::InternalError("GPU processing is disabled.");
 #endif  // !MEDIAPIPE_DISABLE_GPU
   } else {
+#if !MEDIAPIPE_DISABLE_OPENCV
     MP_RETURN_IF_ERROR(RenderCpu(cc));
+#else
+    return absl::InternalError("OpenCV processing is disabled.");
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
   }
 
   return absl::OkStatus();
@@ -181,24 +188,25 @@ absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) {
 }
 
 absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
+#if !MEDIAPIPE_DISABLE_OPENCV
   // Setup source images.
   const auto& current_frame = cc->Inputs().Tag(kCurrentMaskTag).Get<Image>();
-  const cv::Mat current_mat = mediapipe::formats::MatView(&current_frame);
-  RET_CHECK_EQ(current_mat.type(), CV_32FC1)
+  auto current_mat = mediapipe::formats::MatView(&current_frame);
+  RET_CHECK_EQ(current_mat->type(), CV_32FC1)
       << "Only 1-channel float input image is supported.";
 
   const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
-  const cv::Mat previous_mat = mediapipe::formats::MatView(&previous_frame);
-  RET_CHECK_EQ(previous_mat.type(), current_mat.type())
-      << "Warning: mixing input format types: " << previous_mat.type()
-      << " != " << previous_mat.type();
+  auto previous_mat = mediapipe::formats::MatView(&previous_frame);
+  RET_CHECK_EQ(previous_mat->type(), current_mat->type())
+      << "Warning: mixing input format types: " << previous_mat->type()
+      << " != " << previous_mat->type();
 
-  RET_CHECK_EQ(current_mat.rows, previous_mat.rows);
-  RET_CHECK_EQ(current_mat.cols, previous_mat.cols);
+  RET_CHECK_EQ(current_mat->rows, previous_mat->rows);
+  RET_CHECK_EQ(current_mat->cols, previous_mat->cols);
 
   // Setup destination image.
   auto output_frame = std::make_shared<ImageFrame>(
-      current_frame.image_format(), current_mat.cols, current_mat.rows);
+      current_frame.image_format(), current_mat->cols, current_mat->rows);
   cv::Mat output_mat = mediapipe::formats::MatView(output_frame.get());
   output_mat.setTo(cv::Scalar(0));
 
@@ -233,8 +241,8 @@ absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
   // Write directly to the first channel of output.
   for (int i = 0; i < output_mat.rows; ++i) {
     float* out_ptr = output_mat.ptr<float>(i);
-    const float* curr_ptr = current_mat.ptr<float>(i);
-    const float* prev_ptr = previous_mat.ptr<float>(i);
+    const float* curr_ptr = current_mat->ptr<float>(i);
+    const float* prev_ptr = previous_mat->ptr<float>(i);
     for (int j = 0; j < output_mat.cols; ++j) {
       const float new_mask_value = curr_ptr[j];
       const float prev_mask_value = prev_ptr[j];
@@ -245,6 +253,7 @@ absl::Status SegmentationSmoothingCalculator::RenderCpu(CalculatorContext* cc) {
   cc->Outputs()
       .Tag(kOutputMaskTag)
       .AddPacket(MakePacket<Image>(output_frame).At(cc->InputTimestamp()));
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 
   return absl::OkStatus();
 }

mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc

@@ -116,8 +116,8 @@ void RunGraph(Packet curr_packet, Packet prev_packet, bool use_gpu, float ratio,
   ASSERT_EQ(1, output_packets.size());
 
   Image result_image = output_packets[0].Get<Image>();
-  cv::Mat result_mat = formats::MatView(&result_image);
-  result_mat.copyTo(*result);
+  auto result_mat = formats::MatView(&result_image);
+  result_mat->copyTo(*result);
 
   // Fully close graph at end, otherwise calculator+Images are destroyed
   // after calling WaitUntilDone().
@@ -135,10 +135,10 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
 
   Packet curr_packet = MakePacket<Image>(std::make_unique<ImageFrame>(
       ImageFormat::VEC32F1, curr_mat.size().width, curr_mat.size().height));
-  curr_mat.copyTo(formats::MatView(&(curr_packet.Get<Image>())));
+  curr_mat.copyTo(*formats::MatView(&(curr_packet.Get<Image>())));
   Packet prev_packet = MakePacket<Image>(std::make_unique<ImageFrame>(
       ImageFormat::VEC32F1, prev_mat.size().width, prev_mat.size().height));
-  prev_mat.copyTo(formats::MatView(&(prev_packet.Get<Image>())));
+  prev_mat.copyTo(*formats::MatView(&(prev_packet.Get<Image>())));
 
   cv::Mat result;
   RunGraph(curr_packet, prev_packet, use_gpu, mix_ratio, &result);

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -53,7 +53,7 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 // The alpha channel can be set to a single value, or come from an image mask.
 // If the input image has an alpha channel, it will be updated.
 // If the input image doesn't have an alpha channel, one will be added.
-// Adding alpha channel to a Grayscale (single channel) input is not suported.
+// Adding alpha channel to a Grayscale (single channel) input is not supported.
 //
 // Inputs:
 //   One of the following two IMAGE tags:

mediapipe/calculators/image/warp_affine_calculator.cc

@@ -24,7 +24,9 @@
 #endif  // !MEDIAPIPE_DISABLE_GPU
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
+#if !MEDIAPIPE_DISABLE_OPENCV
 #include "mediapipe/calculators/image/affine_transformation_runner_opencv.h"
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 #include "mediapipe/calculators/image/warp_affine_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -54,6 +56,7 @@ AffineTransformation::BorderMode GetBorderMode(
 template <typename ImageT>
 class WarpAffineRunnerHolder {};
 
+#if !MEDIAPIPE_DISABLE_OPENCV
 template <>
 class WarpAffineRunnerHolder<ImageFrame> {
  public:
@@ -69,6 +72,7 @@ class WarpAffineRunnerHolder<ImageFrame> {
  private:
   std::unique_ptr<RunnerType> runner_;
 };
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 
 #if !MEDIAPIPE_DISABLE_GPU
 template <>
@@ -113,7 +117,9 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
                                                      mediapipe::Image> {
    public:
     absl::Status Open(CalculatorContext* cc) {
+#if !MEDIAPIPE_DISABLE_OPENCV
       MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 #if !MEDIAPIPE_DISABLE_GPU
       MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
 #endif  // !MEDIAPIPE_DISABLE_GPU
@@ -133,20 +139,26 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
         return absl::UnavailableError("GPU support is disabled");
 #endif  // !MEDIAPIPE_DISABLE_GPU
       }
+#if !MEDIAPIPE_DISABLE_OPENCV
       ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
       const auto& frame_ptr = input.GetImageFrameSharedPtr();
       // Wrap image into image frame.
       const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
                                    frame_ptr->Height(), frame_ptr->WidthStep(),
                                    const_cast<uint8_t*>(frame_ptr->PixelData()),
-                                   [](uint8* data) {});
+                                   [](uint8* data){});
       ASSIGN_OR_RETURN(auto result,
                        runner->Run(image_frame, matrix, size, border_mode));
       return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
+#else
+      return absl::UnavailableError("OpenCV support is disabled");
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
     }
 
    private:
+#if !MEDIAPIPE_DISABLE_OPENCV
     WarpAffineRunnerHolder<ImageFrame> cpu_holder_;
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 #if !MEDIAPIPE_DISABLE_GPU
     WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
 #endif  // !MEDIAPIPE_DISABLE_GPU
@@ -200,8 +212,10 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
 
 }  // namespace
 
+#if !MEDIAPIPE_DISABLE_OPENCV
 MEDIAPIPE_NODE_IMPLEMENTATION(
     WarpAffineCalculatorImpl<WarpAffineCalculatorCpu>);
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 #if !MEDIAPIPE_DISABLE_GPU
 MEDIAPIPE_NODE_IMPLEMENTATION(
     WarpAffineCalculatorImpl<WarpAffineCalculatorGpu>);

mediapipe/calculators/image/warp_affine_calculator.h

@@ -70,11 +70,13 @@ class WarpAffineCalculatorIntf : public mediapipe::api2::NodeIntf {
   static constexpr mediapipe::api2::Output<ImageT> kOutImage{"IMAGE"};
 };
 
+#if !MEDIAPIPE_DISABLE_OPENCV
 class WarpAffineCalculatorCpu : public WarpAffineCalculatorIntf<ImageFrame> {
  public:
   MEDIAPIPE_NODE_INTERFACE(WarpAffineCalculatorCpu, kInImage, kMatrix,
                            kOutputSize, kOutImage);
 };
+#endif  // !MEDIAPIPE_DISABLE_OPENCV
 #if !MEDIAPIPE_DISABLE_GPU
 class WarpAffineCalculatorGpu
     : public WarpAffineCalculatorIntf<mediapipe::GpuBuffer> {

mediapipe/calculators/image/yuv_to_image_calculator.cc

@@ -0,0 +1,123 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+#include "libyuv/convert_argb.h"
+#include "libyuv/video_common.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/yuv_image.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+// Utility function to convert FourCC enum to string, for error messages.
+std::string FourCCToString(libyuv::FourCC fourcc) {
+  char buf[5];
+  buf[0] = (fourcc >> 24) & 0xff;
+  buf[1] = (fourcc >> 16) & 0xff;
+  buf[2] = (fourcc >> 8) & 0xff;
+  buf[3] = (fourcc)&0xff;
+  buf[4] = 0;
+  return std::string(buf);
+}
+}  // namespace
+
+// Converts a `YUVImage` into an RGB `Image` using libyuv.
+//
+// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
+// YV21) format (as per the `fourcc()` property). This covers the most commonly
+// used YUV image formats used on mobile devices. Other formats are not
+// supported and wil result in an `InvalidArgumentError`.
+class YUVToImageCalculator : public Node {
+ public:
+  static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
+  static constexpr Output<Image> kOutput{"IMAGE"};
+
+  MEDIAPIPE_NODE_CONTRACT(kInput, kOutput);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    const auto& yuv_image = *kInput(cc);
+    // Check that the format is supported.
+    auto format = yuv_image.fourcc();
+    if (format != libyuv::FOURCC_NV12 && format != libyuv::FOURCC_NV21 &&
+        format != libyuv::FOURCC_YV12 && format != libyuv::FOURCC_I420) {
+      return absl::InvalidArgumentError(
+          absl::StrFormat("Unsupported YUVImage format: %s. Only NV12, NV21, "
+                          "YV12 and I420 (aka YV21) are supported.",
+                          FourCCToString(format)));
+    }
+    // Build a transient ImageFrameSharedPtr with default alignment to host
+    // conversion results.
+    ImageFrameSharedPtr image_frame = std::make_shared<ImageFrame>(
+        ImageFormat::SRGB, yuv_image.width(), yuv_image.height());
+    // Perform actual conversion.
+    switch (format) {
+      case libyuv::FOURCC_NV12:
+        // 8-bit Y plane followed by an interleaved 8-bit U/V plane with 2×2
+        // subsampling.
+        libyuv::NV12ToRAW(
+            yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
+            yuv_image.stride(1), image_frame->MutablePixelData(),
+            image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
+        break;
+      case libyuv::FOURCC_NV21:
+        // 8-bit Y plane followed by an interleaved 8-bit V/U plane with 2×2
+        // subsampling.
+        libyuv::NV21ToRAW(
+            yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
+            yuv_image.stride(1), image_frame->MutablePixelData(),
+            image_frame->WidthStep(), yuv_image.width(), yuv_image.height());
+        break;
+      case libyuv::FOURCC_I420:
+        // Also known as YV21.
+        // 8-bit Y plane followed by 8-bit 2×2 subsampled U and V planes.
+        libyuv::I420ToRAW(
+            yuv_image.data(0), yuv_image.stride(0), yuv_image.data(1),
+            yuv_image.stride(1), yuv_image.data(2), yuv_image.stride(2),
+            image_frame->MutablePixelData(), image_frame->WidthStep(),
+            yuv_image.width(), yuv_image.height());
+        break;
+      case libyuv::FOURCC_YV12:
+        // 8-bit Y plane followed by 8-bit 2×2 subsampled V and U planes.
+        libyuv::I420ToRAW(
+            yuv_image.data(0), yuv_image.stride(0), yuv_image.data(2),
+            yuv_image.stride(2), yuv_image.data(1), yuv_image.stride(1),
+            image_frame->MutablePixelData(), image_frame->WidthStep(),
+            yuv_image.width(), yuv_image.height());
+        break;
+      default:
+        // This should never happen (caught by checks above).
+        return absl::InternalError("Unsupported YUVImage format.");
+    }
+    // Finally, build and send an Image object that takes ownership of the
+    // transient ImageFrameSharedPtr object.
+    kOutput(cc).Send(std::make_unique<Image>(std::move(image_frame)));
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(YUVToImageCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/BUILD

@@ -28,7 +28,9 @@ package(default_visibility = ["//visibility:private"])
 
 exports_files(
     glob(["testdata/image_to_tensor/*"]),
-    visibility = ["//mediapipe/calculators/image:__subpackages__"],
+    visibility = [
+        "//mediapipe/calculators/image:__subpackages__",
+    ],
 )
 
 selects.config_setting_group(
@@ -38,6 +40,63 @@ selects.config_setting_group(
     ],
 )
 
+mediapipe_proto_library(
+    name = "audio_to_tensor_calculator_proto",
+    srcs = ["audio_to_tensor_calculator.proto"],
+    visibility = [
+        "//mediapipe/framework:mediapipe_internal",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "audio_to_tensor_calculator",
+    srcs = ["audio_to_tensor_calculator.cc"],
+    visibility = [
+        "//mediapipe/framework:mediapipe_internal",
+    ],
+    deps = [
+        ":audio_to_tensor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_audio_tools//audio/dsp:resampler_q",
+        "@org_tensorflow//tensorflow/lite/c:common",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "audio_to_tensor_calculator_test",
+    srcs = ["audio_to_tensor_calculator_test.cc"],
+    deps = [
+        ":audio_to_tensor_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/strings",
+        "@com_google_audio_tools//audio/dsp:resampler_q",
+        "@org_tensorflow//tensorflow/lite/c:common",
+    ],
+)
+
 mediapipe_proto_library(
     name = "inference_calculator_proto",
     srcs = ["inference_calculator.proto"],
@@ -48,6 +107,14 @@ mediapipe_proto_library(
     ],
 )
 
+# This target defines the "InferenceCalculator" component, which looks for the available concrete
+# implementations linked into the current binary and picks the one to use.
+# You can depend on :inference_calculator instead if you want to automatically include a default
+# set of implementations tailored for the current build configuration.
+# If you want to have precise control of which implementations to include (e.g. for strict binary
+# size concerns), depend on those implementations directly, and do not depend on
+# :inference_calculator.
+# In all cases, use "InferenceCalulator" in your graphs.
 cc_library(
     name = "inference_calculator_interface",
     srcs = ["inference_calculator.cc"],
@@ -60,19 +127,21 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
-        ":inference_calculator_cc_proto",
+        ":inference_calculator_options_lib",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
         "//mediapipe/framework/tool:subgraph_expansion",
-        "//mediapipe/util/tflite:config",
         "//mediapipe/util/tflite:tflite_model_loader",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/lite:framework",
+        "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
     ],
     alwayslink = 1,
@@ -82,16 +151,31 @@ cc_library(
     name = "inference_calculator_gl",
     srcs = ["inference_calculator_gl.cc"],
     tags = ["nomac"],  # config problem with cpuinfo via TF
+    visibility = ["//visibility:public"],
     deps = [
-        "inference_calculator_interface",
+        ":inference_calculator_interface",
         "//mediapipe/gpu:gl_calculator_helper",
-        "//mediapipe/gpu:gpu_buffer",
-        "//mediapipe/util/tflite:tflite_gpu_runner",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@org_tensorflow//tensorflow/lite:framework_stable",
         "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
-        "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
-        "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
-        "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
-        "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "inference_calculator_gl_advanced",
+    srcs = ["inference_calculator_gl_advanced.cc"],
+    tags = ["nomac"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":inference_calculator_interface",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/gpu:gl_calculator_helper",
+        "//mediapipe/util/tflite:tflite_gpu_runner",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
+        "@org_tensorflow//tensorflow/lite:framework_stable",
     ],
     alwayslink = 1,
 )
@@ -108,6 +192,7 @@ cc_library(
         "-framework MetalKit",
     ],
     tags = ["ios"],
+    visibility = ["//visibility:public"],
     deps = [
         "inference_calculator_interface",
         "//mediapipe/gpu:MPPMetalHelper",
@@ -137,10 +222,13 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_interface",
         "@com_google_absl//absl/memory",
         "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
+        "@org_tensorflow//tensorflow/lite:framework_stable",
+        "@org_tensorflow//tensorflow/lite/c:c_api_types",
     ] + select({
         "//conditions:default": [
             "//mediapipe/util:cpu_util",
@@ -154,9 +242,13 @@ cc_library(
 
 cc_library(
     name = "inference_calculator_gl_if_compute_shader_available",
-    deps = select({
+    visibility = ["//visibility:public"],
+    deps = selects.with_or({
         ":compute_shader_unavailable": [],
-        "//conditions:default": [":inference_calculator_gl"],
+        "//conditions:default": [
+            ":inference_calculator_gl",
+            ":inference_calculator_gl_advanced",
+        ],
     }),
 )
 
@@ -303,7 +395,7 @@ cc_library(
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
         "//mediapipe/framework/port:ret_check",
-    ] + select({
+    ] + selects.with_or({
         ":compute_shader_unavailable": [],
         "//conditions:default": [":tensors_to_detections_calculator_gpu_deps"],
     }),
@@ -356,6 +448,57 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_to_tensor_calculator_proto",
+    srcs = ["landmarks_to_tensor_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_to_tensor_calculator",
+    srcs = ["landmarks_to_tensor_calculator.cc"],
+    hdrs = ["landmarks_to_tensor_calculator.h"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":landmarks_to_tensor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "landmarks_to_tensor_calculator_test",
+    srcs = ["landmarks_to_tensor_calculator_test.cc"],
+    deps = [
+        ":landmarks_to_tensor_calculator",
+        ":landmarks_to_tensor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 mediapipe_proto_library(
     name = "tensors_to_floats_calculator_proto",
     srcs = ["tensors_to_floats_calculator.proto"],
@@ -426,6 +569,7 @@ cc_library(
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/formats:tensor",
+        "//mediapipe/util:label_map_cc_proto",
         "//mediapipe/util:resource_util",
     ] + select({
         "//mediapipe:android": [
@@ -448,6 +592,7 @@ mediapipe_proto_library(
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util:label_map_proto",
     ],
 )
 
@@ -509,7 +654,7 @@ cc_library(
 
 cc_library(
     name = "image_to_tensor_calculator_gpu_deps",
-    deps = select({
+    deps = selects.with_or({
         "//mediapipe:android": [
             ":image_to_tensor_converter_gl_buffer",
             "//mediapipe/gpu:gl_calculator_helper",
@@ -614,6 +759,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_converter",
         ":image_to_tensor_utils",
@@ -633,7 +779,7 @@ cc_library(
     name = "image_to_tensor_converter_gl_buffer",
     srcs = ["image_to_tensor_converter_gl_buffer.cc"],
     hdrs = ["image_to_tensor_converter_gl_buffer.h"],
-    deps = ["//mediapipe/framework:port"] + select({
+    deps = ["//mediapipe/framework:port"] + selects.with_or({
         "//mediapipe:apple": [],
         "//conditions:default": [
             ":image_to_tensor_converter",
@@ -800,9 +946,7 @@ cc_library(
         "@com_google_absl//absl/types:span",
         "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:image_frame",
-        "//mediapipe/framework/formats:image_opencv",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",
@@ -832,6 +976,12 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util",
         ],
+    }) + select({
+        "//mediapipe/framework/port:disable_opencv": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:image_opencv",
+            "//mediapipe/framework/port:opencv_imgproc",
+        ],
     }),
     alwayslink = 1,
 )

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -0,0 +1,401 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <algorithm>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/str_format.h"
+#include "audio/dsp/resampler_q.h"
+#include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// Converts audio buffers into tensors, possibly with resampling, buffering
+// and framing, according to specified inputs and options. All input audio
+// buffers will be first resampled from the input sample rate to the target
+// sample rate if they are not equal. The resampled audio data (with the
+// buffered samples from the previous runs in the streaming mode) will be broken
+// into fixed-sized, possibly overlapping frames. Finally, all frames will be
+// converted to and outputted as MediaPipe Tensors. The last output tensor will
+// be zero-padding if the remaining samples are insufficient.
+//
+// This calculator assumes that the input timestamps refer to the first
+// sample in each Matrix. The output timestamps follow this same convention.
+// One Process() call may output multiple tensors packets. The timestamps of
+// the output packets are determined by the timestamp of the previous output
+// packet, the target sample rate, and the number of samples advanced after the
+// previous output.
+//
+// The calculator has two running modes:
+//   Streaming mode: when "streaming_mode" is set to true in the calculator
+//     options, the calculator treats the input audio stream as a continuous
+//     stream. Thus, any samples that are not consumed in the previous runs will
+//     be cached in a global sample buffer. The audio data resampled from the
+//     current raw audio input will be appended to the global sample buffer.
+//     The calculator will process the global sample buffer and output as many
+//     tensors as possible.
+//   Non-streaming mode: when "streaming_mode" is set to false in the calculator
+//     options, the calculators treats the packets in the input audio stream as
+//     a batch of unrelated audio buffers. In each Process() call, the input
+//     buffer will be frist resampled, and framed as fixed-sized, possibly
+//     overlapping tensors. The last tensor produced by a Process() invocation
+//     will be zero-padding if the remaining samples are insufficient. As the
+//     calculator treats the input packets as unrelated, all samples will be
+//     processed immediately and no samples will be cached in the global sample
+//     buffer.
+//
+// Inputs:
+//   AUDIO - mediapipe::Matrix
+//     The audio data represented as mediapipe::Matrix.
+//   SAMPLE_RATE - double @Optional
+//     The sample rate of the corresponding audio data in the "AUDIO" stream.
+//     If a sample rate packet is provided at Timestamp::PreStream(), the sample
+//     rate will be used as the sample rate of every audio packets in the
+//     "AUDIO" stream. Note that one and only one of the "AUDIO" stream's time
+//     series header or the "SAMPLE_RATE" stream can exist.
+//
+// Outputs:
+//   TENSORS - std::vector<Tensor>
+//     Vector containing a single Tensor that represents a fix-sized audio
+//     frame.
+//   TIMESTAMPS - std::vector<Timestamp> @Optional
+//     Vector containing the output timestamps emitted by the current Process()
+//     invocation. In the non-streaming mode, the vector contains all of the
+//     output timestamps for an input audio buffer.
+//
+// Example:
+// node {
+//   calculator: "AudioToTensorCalculator"
+//   input_stream: "AUDIO:audio"
+//   output_stream: "TENSORS:tensors"
+//   output_stream: "TIMESTAMPS:timestamps"
+//   options {
+//     [mediapipe.AudioToTensorCalculatorOptions.ext] {
+//       num_channels: 2
+//       num_samples: 512
+//       num_overlapping_samples: 64
+//       target_sample_rate: 16000
+//       streaming_mode: true # or false
+//     }
+//   }
+// }
+class AudioToTensorCalculator : public Node {
+ public:
+  static constexpr Input<Matrix> kAudioIn{"AUDIO"};
+  // TODO: Removes this optional input stream when the "AUDIO" stream
+  // uses the new mediapipe audio data containers that carry audio metatdata,
+  // such as sample rate.
+  static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
+  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
+  // A vector of the output timestamps emitted by the current Process()
+  // invocation. The packet timestamp is the last emitted timestamp.
+  static constexpr Output<std::vector<Timestamp>>::Optional kTimestampsOut{
+      "TIMESTAMPS"};
+  MEDIAPIPE_NODE_CONTRACT(kAudioIn, kAudioSampleRateIn, kTensorsOut,
+                          kTimestampsOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc);
+  absl::Status Process(CalculatorContext* cc);
+  absl::Status Close(CalculatorContext* cc);
+
+ private:
+  // The target number of channels.
+  int num_channels_;
+  // The target number of samples per channel.
+  int num_samples_;
+  // The number of samples per channel to advance after the current frame is
+  // processed.
+  int frame_step_;
+  bool streaming_mode_;
+  bool check_inconsistent_timestamps_;
+  Timestamp initial_timestamp_ = Timestamp::Unstarted();
+  int64 cumulative_input_samples_ = 0;
+  Timestamp next_output_timestamp_ = Timestamp::Unstarted();
+
+  double source_sample_rate_ = -1;
+  double target_sample_rate_ = -1;
+  // TODO: Configures QResamplerParams through calculator options.
+  audio_dsp::QResamplerParams params_;
+  // A QResampler instance to resample an audio stream.
+  std::unique_ptr<audio_dsp::QResampler<float>> resampler_;
+  Matrix sample_buffer_;
+  int processed_buffer_cols_ = 0;
+
+  absl::Status ProcessStreamingData(CalculatorContext* cc);
+  absl::Status ProcessNonStreamingData(CalculatorContext* cc);
+
+  absl::Status SetupStreamingResampler(double input_sample_rate_);
+  void AppendToSampleBuffer(Matrix buffer_to_append);
+
+  absl::StatusOr<std::vector<Tensor>> ConvertToTensor(
+      const Matrix& frame_to_convert);
+  absl::Status OutputTensors(const Matrix& buffer, bool should_flush,
+                             CalculatorContext* cc);
+};
+
+absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
+  const auto& options =
+      cc->Options<mediapipe::AudioToTensorCalculatorOptions>();
+  if (!options.has_num_channels() || !options.has_num_samples() ||
+      !options.has_target_sample_rate()) {
+    return absl::InvalidArgumentError(
+        "AudioToTensorCalculatorOptions must specifiy "
+        "`num_channels`, `num_samples`, and `target_sample_rate`.");
+  }
+  if (options.streaming_mode()) {
+    // Explicitly disables tiemstamp offset to disallow the timestamp bound
+    // from the input streams to be propagated to the output streams.
+    // In the streaming mode, the output timestamp bound is based on
+    // next_output_timestamp_, which can be smaller than the current input
+    // timestamps.
+    cc->SetTimestampOffset(TimestampDiff::Unset());
+  }
+  return absl::OkStatus();
+}
+
+absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
+  const auto& options =
+      cc->Options<mediapipe::AudioToTensorCalculatorOptions>();
+  num_channels_ = options.num_channels();
+  num_samples_ = options.num_samples();
+  if (options.has_num_overlapping_samples()) {
+    RET_CHECK_GE(options.num_overlapping_samples(), 0);
+    RET_CHECK_LT(options.num_overlapping_samples(), num_samples_);
+    frame_step_ = num_samples_ - options.num_overlapping_samples();
+  } else {
+    frame_step_ = num_samples_;
+  }
+  target_sample_rate_ = options.target_sample_rate();
+  streaming_mode_ = options.streaming_mode();
+  if (streaming_mode_) {
+    check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
+    sample_buffer_.resize(num_channels_, Eigen::NoChange);
+  }
+
+  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+            !kAudioIn(cc).Header().IsEmpty())
+      << "Must either specify the time series header of the \"AUDIO\" stream "
+         "or have the \"SAMPLE_RATE\" stream connected.";
+  if (!kAudioIn(cc).Header().IsEmpty()) {
+    mediapipe::TimeSeriesHeader input_header;
+    MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+        kAudioIn(cc).Header(), &input_header));
+    if (streaming_mode_) {
+      MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+    } else {
+      source_sample_rate_ = input_header.sample_rate();
+    }
+  }
+  return absl::OkStatus();
+}
+
+absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
+  if (cc->InputTimestamp() == Timestamp::PreStream()) {
+    double current_source_sample_rate = kAudioSampleRateIn(cc).Get();
+    if (cc->Options<mediapipe::AudioToTensorCalculatorOptions>()
+            .streaming_mode()) {
+      return SetupStreamingResampler(current_source_sample_rate);
+    } else {
+      source_sample_rate_ = current_source_sample_rate;
+      return absl::OkStatus();
+    }
+  }
+  // Sanity checks.
+  const auto& input_frame = kAudioIn(cc).Get();
+  if (input_frame.rows() != num_channels_) {
+    return absl::InvalidArgumentError(absl::StrFormat(
+        "Audio input has %d channel(s) but the model requires %d channel(s).",
+        input_frame.rows(), num_channels_));
+  }
+  if (num_channels_ > 1 && input_frame.IsRowMajor) {
+    return absl::InvalidArgumentError(
+        "The audio data should be stored in column-major.");
+  }
+  return streaming_mode_ ? ProcessStreamingData(cc)
+                         : ProcessNonStreamingData(cc);
+}
+
+absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
+  if (!streaming_mode_) {
+    return absl::OkStatus();
+  }
+  if (resampler_) {
+    Matrix resampled_buffer(num_channels_, 0);
+    resampler_->Flush(&resampled_buffer);
+    AppendToSampleBuffer(std::move(resampled_buffer));
+  }
+  return OutputTensors(sample_buffer_, /*should_flush=*/true, cc);
+}
+
+absl::Status AudioToTensorCalculator::ProcessStreamingData(
+    CalculatorContext* cc) {
+  const auto& input_buffer = kAudioIn(cc).Get();
+  if (initial_timestamp_ == Timestamp::Unstarted()) {
+    initial_timestamp_ = cc->InputTimestamp();
+    next_output_timestamp_ = initial_timestamp_;
+  }
+  if (source_sample_rate_ != -1 && check_inconsistent_timestamps_) {
+    mediapipe::time_series_util::LogWarningIfTimestampIsInconsistent(
+        cc->InputTimestamp(), initial_timestamp_, cumulative_input_samples_,
+        source_sample_rate_);
+    cumulative_input_samples_ += input_buffer.cols();
+  }
+  if (!kAudioSampleRateIn(cc).IsEmpty()) {
+    double current_source_sample_rate = kAudioSampleRateIn(cc).Get();
+    if (resampler_) {
+      RET_CHECK_EQ(current_source_sample_rate, source_sample_rate_);
+    } else {
+      MP_RETURN_IF_ERROR(SetupStreamingResampler(current_source_sample_rate));
+    }
+  }
+
+  if (resampler_) {
+    Matrix resampled_buffer(num_channels_, 0);
+    resampler_->ProcessSamples(input_buffer, &resampled_buffer);
+    AppendToSampleBuffer(std::move(resampled_buffer));
+  } else {
+    // Tries to consume the input matrix first to avoid extra data copy.
+    auto status_or_matrix = kAudioIn(cc).packet().Consume<Matrix>();
+    if (status_or_matrix.ok()) {
+      Matrix local_matrix(num_channels_, 0);
+      local_matrix.swap(*status_or_matrix.value());
+      AppendToSampleBuffer(std::move(local_matrix));
+    } else {
+      AppendToSampleBuffer(input_buffer);
+    }
+  }
+
+  MP_RETURN_IF_ERROR(OutputTensors(sample_buffer_, /*should_flush=*/false, cc));
+  // Removes the processed samples from the global sample buffer.
+  sample_buffer_ = Matrix(sample_buffer_.rightCols(sample_buffer_.cols() -
+                                                   processed_buffer_cols_ - 1));
+  return absl::OkStatus();
+}
+
+absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
+    CalculatorContext* cc) {
+  initial_timestamp_ = cc->InputTimestamp();
+  next_output_timestamp_ = initial_timestamp_;
+  const auto& input_frame = kAudioIn(cc).Get();
+  double source_sample_rate = kAudioSampleRateIn(cc).GetOr(source_sample_rate_);
+
+  if (source_sample_rate != -1 && source_sample_rate != target_sample_rate_) {
+    std::vector<float> resampled = audio_dsp::QResampleSignal<float>(
+        source_sample_rate, target_sample_rate_, num_channels_, params_,
+        input_frame);
+    Eigen::Map<const Matrix> matrix_mapping(resampled.data(), num_channels_,
+                                            resampled.size() / num_channels_);
+    return OutputTensors(matrix_mapping, /*should_flush=*/true, cc);
+  }
+  return OutputTensors(input_frame, /*should_flush=*/true, cc);
+}
+
+absl::Status AudioToTensorCalculator::SetupStreamingResampler(
+    double input_sample_rate) {
+  if (input_sample_rate == source_sample_rate_) {
+    return absl::OkStatus();
+  }
+  source_sample_rate_ = input_sample_rate;
+  if (source_sample_rate_ != target_sample_rate_) {
+    resampler_ = absl::make_unique<audio_dsp::QResampler<float>>(
+        source_sample_rate_, target_sample_rate_, num_channels_, params_);
+    if (!resampler_) {
+      return absl::InternalError("Failed to initialize resampler.");
+    }
+  }
+  return absl::OkStatus();
+}
+
+void AudioToTensorCalculator::AppendToSampleBuffer(Matrix buffer_to_append) {
+  sample_buffer_.conservativeResize(
+      Eigen::NoChange, sample_buffer_.cols() + buffer_to_append.cols());
+  sample_buffer_.rightCols(buffer_to_append.cols()).swap(buffer_to_append);
+}
+
+absl::StatusOr<std::vector<Tensor>> AudioToTensorCalculator::ConvertToTensor(
+    const Matrix& frame_to_convert) {
+  Tensor tensor(Tensor::ElementType::kFloat32,
+                Tensor::Shape({num_channels_, num_samples_}));
+  auto buffer_view = tensor.GetCpuWriteView();
+  if (frame_to_convert.size() < num_channels_ * num_samples_) {
+    std::memset(buffer_view.buffer<float>(), 0, tensor.bytes());
+  }
+  std::memcpy(buffer_view.buffer<float>(), frame_to_convert.data(),
+              frame_to_convert.size() * sizeof(float));
+  std::vector<Tensor> tensor_vector;
+  tensor_vector.push_back(std::move(tensor));
+  return tensor_vector;
+}
+
+absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
+                                                    bool should_flush,
+                                                    CalculatorContext* cc) {
+  int next_frame_first_col = 0;
+  std::vector<Timestamp> timestamps;
+  while ((!streaming_mode_ || !should_flush) &&
+         next_frame_first_col + num_samples_ <= buffer.cols()) {
+    ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
+                                             0, next_frame_first_col,
+                                             num_channels_, num_samples_)));
+    kTensorsOut(cc).Send(std::move(output_tensor), next_output_timestamp_);
+    timestamps.push_back(next_output_timestamp_);
+    next_output_timestamp_ += round(frame_step_ / target_sample_rate_ *
+                                    Timestamp::kTimestampUnitsPerSecond);
+    next_frame_first_col += frame_step_;
+  }
+  if (should_flush && next_frame_first_col < buffer.cols()) {
+    ASSIGN_OR_RETURN(auto output_tensor,
+                     ConvertToTensor(buffer.block(
+                         0, next_frame_first_col, num_channels_,
+                         std::min(num_samples_,
+                                  (int)buffer.cols() - next_frame_first_col))));
+    // In the streaming mode, the flush happens in Close() and a packet at
+    // Timestamp::Max() will be emitted. In the non-streaming mode, each
+    // Process() invocation will process the entire buffer completely.
+    Timestamp timestamp =
+        streaming_mode_ ? Timestamp::Max() : next_output_timestamp_;
+    timestamps.push_back(timestamp);
+    kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
+  }
+  if (kTimestampsOut(cc).IsConnected()) {
+    Timestamp timestamp = timestamps.back();
+    kTimestampsOut(cc).Send(std::move(timestamps), timestamp);
+  }
+  processed_buffer_cols_ = next_frame_first_col - 1;
+  return absl::OkStatus();
+}
+
+MEDIAPIPE_REGISTER_NODE(AudioToTensorCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -0,0 +1,46 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message AudioToTensorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional AudioToTensorCalculatorOptions ext = 448635064;
+  }
+
+  // The required number of channels the output audio tensor has.
+  optional int64 num_channels = 1;
+
+  // The required number of samples per channel the output audio tensor has.
+  optional int64 num_samples = 2;
+
+  // The number of overlapping samples per channel the output audio tensor has.
+  optional int64 num_overlapping_samples = 3 [default = 0];
+
+  // The target number of samples per second (hertz) of the audio buffers that
+  // will be converted into tensors.
+  optional double target_sample_rate = 4;
+
+  // Whether to treat the input audio stream as a continous stream or a batch
+  // of unrelated audio buffers.
+  optional bool streaming_mode = 5 [default = true];
+
+  // Set to false to disable checks for jitter in timestamp values. Useful with
+  // live audio input.
+  optional bool check_inconsistent_timestamps = 6 [default = true];
+}

mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc

@@ -0,0 +1,483 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/substitute.h"
+#include "audio/dsp/resampler_q.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace {
+
+std::unique_ptr<Matrix> CreateTestMatrix(int num_channels, int num_samples,
+                                         int timestamp) {
+  auto matrix = std::make_unique<Matrix>(num_channels, num_samples);
+  for (int c = 0; c < num_channels; ++c) {
+    for (int i = 0; i < num_samples; ++i) {
+      // A float value with the sample, channel, and timestamp separated by a
+      // few orders of magnitude, for easy parsing by humans.
+      (*matrix)(c, i) = timestamp / 10000 + i + c / 100.0;
+    }
+  }
+  return matrix;
+}
+
+std::unique_ptr<Matrix> ResampleBuffer(const Matrix& input_matrix,
+                                       double resampling_factor) {
+  audio_dsp::QResamplerParams params;
+  std::vector<float> resampled;
+  int num_channels = input_matrix.rows();
+  std::vector<float> input_data(input_matrix.data(),
+                                input_matrix.data() + input_matrix.size());
+  resampled = audio_dsp::QResampleSignal<float>(
+      1, resampling_factor, num_channels, params, input_data);
+  Matrix res = Eigen::Map<Matrix>(resampled.data(), num_channels,
+                                  resampled.size() / num_channels);
+  return std::make_unique<Matrix>(std::move(res));
+}
+
+class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
+ protected:
+  void SetUp() override {}
+  void Run(int num_samples, int num_overlapping_samples,
+           double resampling_factor, const Matrix& input_matrix) {
+    double input_sample_rate = 10000;
+    double target_sample_rate = input_sample_rate * resampling_factor;
+    auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        absl::Substitute(R"(
+        input_stream: "audio"
+        input_stream: "sample_rate"
+        output_stream: "tensors"
+        output_stream: "timestamps"
+        node {
+          calculator: "AudioToTensorCalculator"
+          input_stream: "AUDIO:audio"
+          input_stream: "SAMPLE_RATE:sample_rate"
+          output_stream: "TENSORS:tensors"
+          output_stream: "TIMESTAMPS:timestamps"
+          options {
+            [mediapipe.AudioToTensorCalculatorOptions.ext] {
+              num_channels: $0
+              num_samples: $1
+              num_overlapping_samples: $2
+              target_sample_rate: $3
+              streaming_mode: false
+            }
+          }
+        }
+        )",
+                         /*$0=*/input_matrix.rows(),
+                         /*$1=*/num_samples, /*$2=*/num_overlapping_samples,
+                         /*$3=*/target_sample_rate));
+    tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
+    tool::AddVectorSink("timestamps", &graph_config, &timestamps_packets_);
+
+    // Run the graph.
+    MP_ASSERT_OK(graph_.Initialize(graph_config));
+    MP_ASSERT_OK(graph_.StartRun({}));
+    // Run with the input matrix multiple times.
+    for (int i = 0; i < num_iterations_; ++i) {
+      MP_ASSERT_OK(graph_.AddPacketToInputStream(
+          "audio",
+          MakePacket<Matrix>(input_matrix)
+              .At(Timestamp(i * Timestamp::kTimestampUnitsPerSecond))));
+      MP_ASSERT_OK(graph_.AddPacketToInputStream(
+          "sample_rate",
+          MakePacket<double>(input_sample_rate)
+              .At(Timestamp(i * Timestamp::kTimestampUnitsPerSecond))));
+    }
+    MP_ASSERT_OK(graph_.CloseAllInputStreams());
+    MP_ASSERT_OK(graph_.WaitUntilIdle());
+  }
+
+  void CheckTensorsOutputPackets(const Matrix& expected_matrix,
+                                 int sample_offset, int num_tensors_per_input) {
+    ASSERT_EQ(num_iterations_ * num_tensors_per_input, tensors_packets_.size());
+    for (int i = 0; i < num_iterations_; ++i) {
+      for (int j = 0; j < num_tensors_per_input; ++j) {
+        CheckTensorsOutputPacket(
+            expected_matrix, tensors_packets_[i * num_tensors_per_input + j],
+            /*sample_offset*/ sample_offset * j, /*index=*/j);
+      }
+    }
+  }
+
+  void CheckTensorsOutputPacket(const Matrix& expected_matrix,
+                                const Packet& packet, int sample_offset,
+                                int index) {
+    MP_ASSERT_OK(packet.ValidateAsType<std::vector<Tensor>>());
+    ASSERT_EQ(1, packet.Get<std::vector<Tensor>>().size());
+    const Tensor& output_tensor = packet.Get<std::vector<Tensor>>()[0];
+    auto* buffer = output_tensor.GetCpuReadView().buffer<float>();
+    int num_values = output_tensor.shape().num_elements();
+    const std::vector<float> output_floats(buffer, buffer + num_values);
+    for (int i = 0; i < num_values; ++i) {
+      if (i + sample_offset >= expected_matrix.size()) {
+        EXPECT_FLOAT_EQ(output_floats[i], 0);
+      } else {
+        EXPECT_FLOAT_EQ(output_floats[i],
+                        expected_matrix.coeff((i + sample_offset) % 2,
+                                              (i + sample_offset) / 2))
+            << "i=" << i << ", sample_offset=" << sample_offset;
+      }
+    }
+  }
+
+  void CheckTimestampsOutputPackets(
+      std::vector<int64> expected_timestamp_values) {
+    ASSERT_EQ(num_iterations_, timestamps_packets_.size());
+    for (int i = 0; i < timestamps_packets_.size(); ++i) {
+      const auto& p = timestamps_packets_[i];
+      MP_ASSERT_OK(p.ValidateAsType<std::vector<Timestamp>>());
+      auto output_timestamps = p.Get<std::vector<Timestamp>>();
+      int64 base_timestamp = i * Timestamp::kTimestampUnitsPerSecond;
+      std::vector<Timestamp> expected_timestamps;
+      expected_timestamps.resize(expected_timestamp_values.size());
+      std::transform(
+          expected_timestamp_values.begin(), expected_timestamp_values.end(),
+          expected_timestamps.begin(), [base_timestamp](int64 v) -> Timestamp {
+            return Timestamp(v + base_timestamp);
+          });
+      EXPECT_EQ(expected_timestamps, output_timestamps);
+      EXPECT_EQ(p.Timestamp(), expected_timestamps.back());
+    }
+  }
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
+
+ private:
+  CalculatorGraph graph_;
+  int num_iterations_ = 10;
+  std::vector<Packet> tensors_packets_;
+  std::vector<Packet> timestamps_packets_;
+};
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest,
+       ConvertToNoOverlappingFp32Tensors) {
+  auto input_matrix = CreateTestMatrix(2, 8, 0);
+  Run(/*num_samples=*/4, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/1.0f, *input_matrix);
+  CheckTensorsOutputPackets(*input_matrix, /*sample_offset=*/8,
+                            /*num_tensors_per_input=*/2);
+  CheckTimestampsOutputPackets({0, 400});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest,
+       ConvertToOverlappingFp32Tensors) {
+  auto input_matrix = CreateTestMatrix(2, 8, 0);
+  Run(/*num_samples=*/4, /*num_overlapping_samples=*/2,
+      /*resampling_factor=*/1.0f, *input_matrix);
+  CheckTensorsOutputPackets(*input_matrix, /*sample_offset=*/4,
+                            /*num_tensors_per_input=*/4);
+  CheckTimestampsOutputPackets({0, 200, 400, 600});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, TensorsWithZeroPadding) {
+  auto input_matrix = CreateTestMatrix(2, 7, 0);
+  Run(/*num_samples=*/4, /*num_overlapping_samples=*/2,
+      /*resampling_factor=*/1.0f, *input_matrix);
+  CheckTensorsOutputPackets(*input_matrix, /*sample_offset=*/4,
+                            /*num_tensors_per_input=*/3);
+  CheckTimestampsOutputPackets({0, 200, 400});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Downsampling) {
+  auto input_matrix = CreateTestMatrix(2, 1024, 0);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/0.5f, *input_matrix);
+  auto expected_matrix =
+      ResampleBuffer(*input_matrix, /*resampling_factor=*/0.5f);
+  CheckTensorsOutputPackets(*expected_matrix, /*sample_offset=*/512,
+                            /*num_tensors_per_input=*/3);
+  CheckTimestampsOutputPackets({0, 51200, 102400});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest,
+       DownsamplingWithOverlapping) {
+  auto input_matrix = CreateTestMatrix(2, 1024, 0);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/0.5f, *input_matrix);
+  auto expected_matrix =
+      ResampleBuffer(*input_matrix, /*resampling_factor=*/0.5f);
+  CheckTensorsOutputPackets(*expected_matrix, /*sample_offset=*/384,
+                            /*num_tensors_per_input=*/3);
+  CheckTimestampsOutputPackets({0, 38400, 76800});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Upsampling) {
+  auto input_matrix = CreateTestMatrix(2, 1024, 0);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/2.0f, *input_matrix);
+  auto expected_matrix =
+      ResampleBuffer(*input_matrix, /*resampling_factor=*/2.0f);
+  CheckTensorsOutputPackets(*expected_matrix,
+                            /*sample_offset=*/512,
+                            /*num_tensors_per_input=*/9);
+  CheckTimestampsOutputPackets(
+      {0, 12800, 25600, 38400, 51200, 64000, 76800, 89600, 102400});
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, UpsamplingWithOverlapping) {
+  auto input_matrix = CreateTestMatrix(2, 256, 0);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/2.0f, *input_matrix);
+  auto expected_matrix =
+      ResampleBuffer(*input_matrix, /*resampling_factor=*/2.0f);
+  CheckTensorsOutputPackets(*expected_matrix,
+                            /*sample_offset=*/384,
+                            /*num_tensors_per_input=*/3);
+  CheckTimestampsOutputPackets({0, 9600, 19200});
+  CloseGraph();
+}
+
+class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
+ protected:
+  void SetUp() override { sample_buffer_ = std::make_unique<Matrix>(2, 0); }
+
+  void SetInputBufferNumSamplesPerChannel(int num_samples) {
+    input_buffer_num_samples_ = num_samples;
+  }
+
+  void SetNumIterations(int num_iterations) {
+    num_iterations_ = num_iterations;
+  }
+
+  int GetExpectedNumOfSamples() {
+    Matrix* expected_matrix =
+        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
+    return expected_matrix->cols();
+  }
+
+  void Run(int num_samples, int num_overlapping_samples,
+           double resampling_factor) {
+    double input_sample_rate = 10000;
+    double target_sample_rate = input_sample_rate * resampling_factor;
+    auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        absl::Substitute(R"(
+        input_stream: "audio"
+        input_stream: "sample_rate"
+        output_stream: "tensors"
+        node {
+          calculator: "AudioToTensorCalculator"
+          input_stream: "AUDIO:audio"
+          input_stream: "SAMPLE_RATE:sample_rate"
+          output_stream: "TENSORS:tensors"
+          options {
+            [mediapipe.AudioToTensorCalculatorOptions.ext] {
+              num_channels: 2
+              num_samples: $0
+              num_overlapping_samples: $1
+              target_sample_rate: $2
+              streaming_mode:true
+            }
+          }
+        }
+        )",
+                         /*$0=*/num_samples, /*$1=*/num_overlapping_samples,
+                         /*$2=*/target_sample_rate));
+    tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
+
+    // Run the graph.
+    MP_ASSERT_OK(graph_.Initialize(graph_config));
+    MP_ASSERT_OK(graph_.StartRun({}));
+    for (int i = 0; i < num_iterations_; ++i) {
+      Timestamp input_timestamp(Timestamp::kTimestampUnitsPerSecond * i);
+      auto new_data = CreateTestMatrix(2, input_buffer_num_samples_,
+                                       input_timestamp.Value());
+      MP_ASSERT_OK(graph_.AddPacketToInputStream(
+          "audio", MakePacket<Matrix>(*new_data).At(input_timestamp)));
+      MP_ASSERT_OK(graph_.AddPacketToInputStream(
+          "sample_rate",
+          MakePacket<double>(input_sample_rate).At(input_timestamp)));
+      sample_buffer_->conservativeResize(
+          Eigen::NoChange, sample_buffer_->cols() + new_data->cols());
+      sample_buffer_->rightCols(new_data->cols()).swap(*new_data);
+    }
+    MP_ASSERT_OK(graph_.CloseAllInputStreams());
+    MP_ASSERT_OK(graph_.WaitUntilIdle());
+    if (resampling_factor != 1) {
+      resampled_buffer_ = ResampleBuffer(*sample_buffer_, resampling_factor);
+    }
+  }
+
+  void CheckTensorsOutputPackets(int sample_offset, int num_packets,
+                                 int64 timestamp_interval,
+                                 bool output_last_at_close) {
+    ASSERT_EQ(num_packets, tensors_packets_.size());
+    for (int i = 0; i < num_packets; ++i) {
+      if (i == num_packets - 1 && output_last_at_close) {
+        CheckTensorsOutputPacket(sample_offset * i, i, Timestamp::Max());
+      } else {
+        CheckTensorsOutputPacket(sample_offset * i, i,
+                                 Timestamp(timestamp_interval * i));
+      }
+    }
+  }
+
+  void CheckTensorsOutputPacket(int sample_offset, int index,
+                                Timestamp expected_timestamp) {
+    const Packet& p = tensors_packets_[index];
+    MP_ASSERT_OK(p.ValidateAsType<std::vector<Tensor>>());
+    const Tensor& output_tensor = p.Get<std::vector<Tensor>>()[0];
+    auto buffer = output_tensor.GetCpuReadView().buffer<float>();
+    int num_values = output_tensor.shape().num_elements();
+    std::vector<float> output_floats(buffer, buffer + num_values);
+    Matrix* expected_matrix =
+        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
+    for (int i = 0; i < num_values; ++i) {
+      if (i + sample_offset >= expected_matrix->size()) {
+        EXPECT_FLOAT_EQ(output_floats[i], 0);
+      } else {
+        EXPECT_NEAR(output_floats[i],
+                    expected_matrix->coeff((i + sample_offset) % 2,
+                                           (i + sample_offset) / 2),
+                    0.001)
+            << "i=" << i << ", sample_offset=" << sample_offset
+            << ", packet index=" << index;
+      }
+    }
+    EXPECT_EQ(p.Timestamp(), expected_timestamp);
+  }
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
+
+ private:
+  int input_buffer_num_samples_ = 10;
+  int num_iterations_ = 10;
+  CalculatorGraph graph_;
+  std::vector<Packet> tensors_packets_;
+  std::unique_ptr<Matrix> sample_buffer_;
+  std::unique_ptr<Matrix> resampled_buffer_;
+};
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest,
+       OutputNoOverlappingFp32Tensors) {
+  Run(/*num_samples=*/5, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/1.0f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/10,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 5),
+      /*timestamp_interval=*/500,
+      /*output_last_at_close=*/false);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputRemainingInCloseMethod) {
+  Run(/*num_samples=*/6, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/1.0f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/12,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 6),
+      /*timestamp_interval=*/600,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputOverlappingFp32Tensors) {
+  SetInputBufferNumSamplesPerChannel(12);
+  Run(/*num_samples=*/10, /*num_overlapping_samples=*/2,
+      /*resampling_factor=*/1.0f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/16,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 8),
+      /*timestamp_interval=*/800,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, Downsampling) {
+  SetInputBufferNumSamplesPerChannel(1000);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/0.5f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/512,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*timestamp_interval=*/51200,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, DownsamplingWithOverlapping) {
+  SetInputBufferNumSamplesPerChannel(1024);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/0.5f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/384,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*timestamp_interval=*/38400,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, Upsampling) {
+  SetInputBufferNumSamplesPerChannel(1000);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/2.0f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/512,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*timestamp_interval=*/12800,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, UpsamplingWithOverlapping) {
+  SetInputBufferNumSamplesPerChannel(1024);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/2.0f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/384,
+      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*timestamp_interval=*/9600,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest,
+       OnlyOutputInCloseIfNoSufficientSamples) {
+  SetNumIterations(1);
+  Run(/*num_samples=*/8, /*num_overlapping_samples=*/0,
+      /*resampling_factor=*/0.5f);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/0,
+      /*num_packets=*/1,
+      /*timestamp_interval=*/0,
+      /*output_last_at_close=*/true);
+  CloseGraph();
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -49,7 +49,6 @@
 #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
-
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
 namespace mediapipe {
@@ -142,11 +141,37 @@ class ImageToTensorCalculator : public Node {
     const auto& options =
         cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
 
-    RET_CHECK(options.has_output_tensor_float_range())
+    RET_CHECK(options.has_output_tensor_float_range() ||
+              options.has_output_tensor_int_range() ||
+              options.has_output_tensor_uint_range())
         << "Output tensor range is required.";
-    RET_CHECK_LT(options.output_tensor_float_range().min(),
-                 options.output_tensor_float_range().max())
-        << "Valid output tensor range is required.";
+    if (options.has_output_tensor_float_range()) {
+      RET_CHECK_LT(options.output_tensor_float_range().min(),
+                   options.output_tensor_float_range().max())
+          << "Valid output float tensor range is required.";
+    }
+    if (options.has_output_tensor_uint_range()) {
+      RET_CHECK_LT(options.output_tensor_uint_range().min(),
+                   options.output_tensor_uint_range().max())
+          << "Valid output uint tensor range is required.";
+      RET_CHECK_GE(options.output_tensor_uint_range().min(), 0)
+          << "The minimum of the output uint tensor range must be "
+             "non-negative.";
+      RET_CHECK_LE(options.output_tensor_uint_range().max(), 255)
+          << "The maximum of the output uint tensor range must be less than or "
+             "equal to 255.";
+    }
+    if (options.has_output_tensor_int_range()) {
+      RET_CHECK_LT(options.output_tensor_int_range().min(),
+                   options.output_tensor_int_range().max())
+          << "Valid output int tensor range is required.";
+      RET_CHECK_GE(options.output_tensor_int_range().min(), -128)
+          << "The minimum of the output int tensor range must be greater than "
+             "or equal to -128.";
+      RET_CHECK_LE(options.output_tensor_int_range().max(), 127)
+          << "The maximum of the output int tensor range must be less than or "
+             "equal to 127.";
+    }
     RET_CHECK_GT(options.output_tensor_width(), 0)
         << "Valid output tensor width is required.";
     RET_CHECK_GT(options.output_tensor_height(), 0)
@@ -175,9 +200,19 @@ class ImageToTensorCalculator : public Node {
     options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
     output_width_ = options_.output_tensor_width();
     output_height_ = options_.output_tensor_height();
-    range_min_ = options_.output_tensor_float_range().min();
-    range_max_ = options_.output_tensor_float_range().max();
-
+    is_float_output_ = options_.has_output_tensor_float_range();
+    if (options_.has_output_tensor_uint_range()) {
+      range_min_ =
+          static_cast<float>(options_.output_tensor_uint_range().min());
+      range_max_ =
+          static_cast<float>(options_.output_tensor_uint_range().max());
+    } else if (options_.has_output_tensor_int_range()) {
+      range_min_ = static_cast<float>(options_.output_tensor_int_range().min());
+      range_max_ = static_cast<float>(options_.output_tensor_int_range().max());
+    } else {
+      range_min_ = options_.output_tensor_float_range().min();
+      range_max_ = options_.output_tensor_float_range().max();
+    }
     return absl::OkStatus();
   }
 
@@ -225,7 +260,7 @@ class ImageToTensorCalculator : public Node {
     }
 
     // Lazy initialization of the GPU or CPU converter.
-    MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, image->UsesGpu()));
+    MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, *image.get()));
 
     ASSIGN_OR_RETURN(Tensor tensor,
                      (image->UsesGpu() ? gpu_converter_ : cpu_converter_)
@@ -257,6 +292,17 @@ class ImageToTensorCalculator : public Node {
     }
   }
 
+  Tensor::ElementType GetOutputTensorType() {
+    if (is_float_output_) {
+      return Tensor::ElementType::kFloat32;
+    }
+    if (range_min_ < 0) {
+      return Tensor::ElementType::kInt8;
+    } else {
+      return Tensor::ElementType::kUInt8;
+    }
+  }
+
   absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
       CalculatorContext* cc) {
     if (kIn(cc).IsConnected()) {
@@ -283,9 +329,15 @@ class ImageToTensorCalculator : public Node {
     }
   }
 
-  absl::Status InitConverterIfNecessary(CalculatorContext* cc, bool use_gpu) {
+  absl::Status InitConverterIfNecessary(CalculatorContext* cc,
+                                        const Image& image) {
     // Lazy initialization of the GPU or CPU converter.
-    if (use_gpu) {
+    if (image.UsesGpu()) {
+      if (!is_float_output_) {
+        return absl::UnimplementedError(
+            "ImageToTensorConverter for the input GPU image currently doesn't "
+            "support quantization.");
+      }
       if (!gpu_converter_) {
 #if !MEDIAPIPE_DISABLE_GPU
 #if MEDIAPIPE_METAL_ENABLED
@@ -296,17 +348,26 @@ class ImageToTensorCalculator : public Node {
                          CreateImageToGlBufferTensorConverter(
                              cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #else
-        ASSIGN_OR_RETURN(gpu_converter_,
-                         CreateImageToGlTextureTensorConverter(
-                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+        // Check whether the underlying storage object is a GL texture.
+        if (image.GetGpuBuffer()
+                .internal_storage<mediapipe::GlTextureBuffer>()) {
+          ASSIGN_OR_RETURN(
+              gpu_converter_,
+              CreateImageToGlTextureTensorConverter(
+                  cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+        } else {
+          return absl::UnimplementedError(
+              "ImageToTensorConverter for the input GPU image is unavailable.");
+        }
 #endif  // MEDIAPIPE_METAL_ENABLED
 #endif  // !MEDIAPIPE_DISABLE_GPU
       }
     } else {
       if (!cpu_converter_) {
 #if !MEDIAPIPE_DISABLE_OPENCV
-        ASSIGN_OR_RETURN(cpu_converter_,
-                         CreateOpenCvConverter(cc, GetBorderMode()));
+        ASSIGN_OR_RETURN(
+            cpu_converter_,
+            CreateOpenCvConverter(cc, GetBorderMode(), GetOutputTensorType()));
 #else
         LOG(FATAL) << "Cannot create image to tensor opencv converter since "
                       "MEDIAPIPE_DISABLE_OPENCV is defined.";
@@ -321,6 +382,7 @@ class ImageToTensorCalculator : public Node {
   mediapipe::ImageToTensorCalculatorOptions options_;
   int output_width_ = 0;
   int output_height_ = 0;
+  bool is_float_output_ = false;
   float range_min_ = 0.0f;
   float range_max_ = 1.0f;
 };

mediapipe/calculators/tensor/image_to_tensor_calculator.proto

@@ -31,6 +31,22 @@ message ImageToTensorCalculatorOptions {
     optional float max = 2;
   }
 
+  // Range of int values [min, max].
+  // min, must be strictly less than max.
+  // Please note that IntRange is supported for CPU tensors only.
+  message IntRange {
+    optional int64 min = 1;
+    optional int64 max = 2;
+  }
+
+  // Range of uint values [min, max].
+  // min, must be strictly less than max.
+  // Please note that UIntRange is supported for CPU tensors only.
+  message UIntRange {
+    optional uint64 min = 1;
+    optional uint64 max = 2;
+  }
+
   // Pixel extrapolation methods. See @border_mode.
   enum BorderMode {
     BORDER_UNSPECIFIED = 0;
@@ -49,6 +65,8 @@ message ImageToTensorCalculatorOptions {
   // Output tensor element range/type image pixels are converted to.
   oneof range {
     FloatRange output_tensor_float_range = 4;
+    IntRange output_tensor_int_range = 7;
+    UIntRange output_tensor_uint_range = 8;
   }
 
   // For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -61,7 +61,8 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
                                  float range_max, int tensor_width,
                                  int tensor_height, bool keep_aspect,
                                  absl::optional<BorderMode> border_mode,
-                                 const mediapipe::NormalizedRect& roi) {
+                                 const mediapipe::NormalizedRect& roi,
+                                 bool output_int_tensor) {
   std::string border_mode_str;
   if (border_mode) {
     switch (*border_mode) {
@@ -73,6 +74,30 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
         break;
     }
   }
+  std::string output_tensor_range;
+  if (output_int_tensor) {
+    if (range_min < 0) {
+      output_tensor_range = absl::Substitute(R"(output_tensor_int_range {
+                min: $0
+                max: $1
+              })",
+                                             static_cast<int>(range_min),
+                                             static_cast<int>(range_max));
+    } else {
+      output_tensor_range = absl::Substitute(R"(output_tensor_uint_range {
+                min: $0
+                max: $1
+              })",
+                                             static_cast<uint>(range_min),
+                                             static_cast<uint>(range_max));
+    }
+  } else {
+    output_tensor_range = absl::Substitute(R"(output_tensor_float_range {
+                min: $0
+                max: $1
+              })",
+                                           range_min, range_max);
+  }
   auto graph_config = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
       absl::Substitute(R"(
         input_stream: "input_image"
@@ -86,22 +111,18 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
             [mediapipe.ImageToTensorCalculatorOptions.ext] {
               output_tensor_width: $0
               output_tensor_height: $1
-              keep_aspect_ratio: $4
-              output_tensor_float_range {
-                min: $2
-                max: $3
-              }
-              $5 # border mode
+              keep_aspect_ratio: $2
+              $3 # output range
+              $4 # border mode
             }
           }
         }
         )",
                        /*$0=*/tensor_width,
                        /*$1=*/tensor_height,
-                       /*$2=*/range_min,
-                       /*$3=*/range_max,
-                       /*$4=*/keep_aspect ? "true" : "false",
-                       /*$5=*/border_mode_str));
+                       /*$2=*/keep_aspect ? "true" : "false",
+                       /*$3=*/output_tensor_range,
+                       /*$4=*/border_mode_str));
 
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor", &graph_config, &output_packets);
@@ -126,11 +147,24 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
   ASSERT_THAT(tensor_vec, testing::SizeIs(1));
 
   const Tensor& tensor = tensor_vec[0];
-  EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
-
   auto view = tensor.GetCpuReadView();
-  cv::Mat tensor_mat(tensor_height, tensor_width, CV_32FC3,
-                     const_cast<float*>(view.buffer<float>()));
+  cv::Mat tensor_mat;
+  if (output_int_tensor) {
+    if (range_min < 0) {
+      EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kInt8);
+      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8SC3,
+                           const_cast<int8*>(view.buffer<int8>()));
+    } else {
+      EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kUInt8);
+      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8UC3,
+                           const_cast<uint8*>(view.buffer<uint8>()));
+    }
+  } else {
+    EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
+    tensor_mat = cv::Mat(tensor_height, tensor_width, CV_32FC3,
+                         const_cast<float*>(view.buffer<float>()));
+  }
+
   cv::Mat result_rgb;
   auto transformation =
       GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f).value();
@@ -170,16 +204,29 @@ enum class InputType { kImageFrame, kImage };
 const std::vector<InputType> kInputTypesToTest = {InputType::kImageFrame,
                                                   InputType::kImage};
 
-void RunTest(cv::Mat input, cv::Mat expected_result, float range_min,
-             float range_max, int tensor_width, int tensor_height,
-             bool keep_aspect, absl::optional<BorderMode> border_mode,
+void RunTest(cv::Mat input, cv::Mat expected_result,
+             std::vector<std::pair<float, float>> float_ranges,
+             std::vector<std::pair<int, int>> int_ranges, int tensor_width,
+             int tensor_height, bool keep_aspect,
+             absl::optional<BorderMode> border_mode,
              const mediapipe::NormalizedRect& roi) {
   for (auto input_type : kInputTypesToTest) {
-    RunTestWithInputImagePacket(
-        input_type == InputType::kImageFrame ? MakeImageFramePacket(input)
-                                             : MakeImagePacket(input),
-        expected_result, range_min, range_max, tensor_width, tensor_height,
-        keep_aspect, border_mode, roi);
+    for (auto float_range : float_ranges) {
+      RunTestWithInputImagePacket(
+          input_type == InputType::kImageFrame ? MakeImageFramePacket(input)
+                                               : MakeImagePacket(input),
+          expected_result, float_range.first, float_range.second, tensor_width,
+          tensor_height, keep_aspect, border_mode, roi,
+          /*output_int_tensor=*/false);
+    }
+    for (auto int_range : int_ranges) {
+      RunTestWithInputImagePacket(
+          input_type == InputType::kImageFrame ? MakeImageFramePacket(input)
+                                               : MakeImagePacket(input),
+          expected_result, int_range.first, int_range.second, tensor_width,
+          tensor_height, keep_aspect, border_mode, roi,
+          /*output_int_tensor=*/true);
+    }
   }
 }
 
@@ -195,8 +242,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
              "tensor/testdata/image_to_tensor/input.jpg"),
       GetRgb("/mediapipe/calculators/"
              "tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
-      /*range_min=*/0.0f,
-      /*range_max=*/1.0f,
+      /*float_ranges=*/{{0.0f, 1.0f}},
+      /*int_ranges=*/{{0, 255}, {-128, 127}},
       /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
       /*border mode*/ {}, roi);
 }
@@ -213,8 +260,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "medium_sub_rect_keep_aspect_border_zero.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
           BorderMode::kZero, roi);
 }
@@ -231,7 +278,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "medium_sub_rect_keep_aspect_with_rotation.png"),
-          /*range_min=*/0.0f, /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
           BorderMode::kReplicate, roi);
 }
@@ -249,7 +297,8 @@ TEST(ImageToTensorCalculatorTest,
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "medium_sub_rect_keep_aspect_with_rotation_border_zero.png"),
-          /*range_min=*/0.0f, /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
           BorderMode::kZero, roi);
 }
@@ -267,8 +316,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
       GetRgb(
           "/mediapipe/calculators/"
           "tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
-      /*range_min=*/-1.0f,
-      /*range_max=*/1.0f,
+      /*float_ranges=*/{{-1.0f, 1.0f}},
+      /*int_ranges=*/{{0, 255}, {-128, 127}},
       /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
       BorderMode::kReplicate, roi);
 }
@@ -285,8 +334,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "medium_sub_rect_with_rotation_border_zero.png"),
-          /*range_min=*/-1.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{-1.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
           BorderMode::kZero, roi);
 }
@@ -302,8 +351,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRect) {
                  "tensor/testdata/image_to_tensor/input.jpg"),
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/large_sub_rect.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
           BorderMode::kReplicate, roi);
 }
@@ -320,8 +369,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) {
              "tensor/testdata/image_to_tensor/input.jpg"),
       GetRgb("/mediapipe/calculators/"
              "tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"),
-      /*range_min=*/0.0f,
-      /*range_max=*/1.0f,
+      /*float_ranges=*/{{0.0f, 1.0f}},
+      /*int_ranges=*/{{0, 255}, {-128, 127}},
       /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
       BorderMode::kZero, roi);
 }
@@ -338,8 +387,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
              "tensor/testdata/image_to_tensor/input.jpg"),
       GetRgb("/mediapipe/calculators/"
              "tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
-      /*range_min=*/0.0f,
-      /*range_max=*/1.0f,
+      /*float_ranges=*/{{0.0f, 1.0f}},
+      /*int_ranges=*/{{0, 255}, {-128, 127}},
       /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
       BorderMode::kReplicate, roi);
 }
@@ -356,8 +405,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "large_sub_rect_keep_aspect_border_zero.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
           BorderMode::kZero, roi);
 }
@@ -374,8 +423,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "large_sub_rect_keep_aspect_with_rotation.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
           /*border_mode=*/{}, roi);
 }
@@ -393,8 +442,8 @@ TEST(ImageToTensorCalculatorTest,
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/"
                  "large_sub_rect_keep_aspect_with_rotation_border_zero.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
           /*border_mode=*/BorderMode::kZero, roi);
 }
@@ -410,8 +459,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
                   "tensor/testdata/image_to_tensor/input.jpg"),
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/noop_except_range.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
           BorderMode::kReplicate, roi);
 }
@@ -427,8 +476,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) {
                   "tensor/testdata/image_to_tensor/input.jpg"),
           GetRgb("/mediapipe/calculators/"
                  "tensor/testdata/image_to_tensor/noop_except_range.png"),
-          /*range_min=*/0.0f,
-          /*range_max=*/1.0f,
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
           BorderMode::kZero, roi);
 }

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -268,10 +268,12 @@ class GlProcessor : public ImageToTensorConverter {
                                  const RotatedRect& roi,
                                  const Size& output_dims, float range_min,
                                  float range_max) override {
-    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
-      return InvalidArgumentError(
-          absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ",
-                       static_cast<uint32_t>(input.format())));
+    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+      return InvalidArgumentError(absl::StrCat(
+          "Only 4-channel texture input formats are supported, passed format: ",
+          static_cast<uint32_t>(input.format())));
     }
 
     constexpr int kNumChannels = 3;

mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc

@@ -16,7 +16,7 @@
 
 #include "mediapipe/framework/port.h"
 
-#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #include <array>
 #include <memory>
@@ -172,10 +172,12 @@ class GlProcessor : public ImageToTensorConverter {
                                  const RotatedRect& roi,
                                  const Size& output_dims, float range_min,
                                  float range_max) override {
-    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
-      return InvalidArgumentError(
-          absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ",
-                       static_cast<uint32_t>(input.format())));
+    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+      return InvalidArgumentError(absl::StrCat(
+          "Only 4-channel texture input formats are supported, passed format: ",
+          static_cast<uint32_t>(input.format())));
     }
 
     constexpr int kNumChannels = 3;
@@ -339,4 +341,4 @@ CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
 
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30

mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h

@@ -17,7 +17,7 @@
 
 #include "mediapipe/framework/port.h"
 
-#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #include <memory>
 
@@ -37,6 +37,6 @@ CreateImageToGlTextureTensorConverter(CalculatorContext* cc,
 
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #endif  // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_

mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc

@@ -2,7 +2,7 @@
 
 #include "mediapipe/framework/port.h"
 
-#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #include <array>
 #include <memory>
@@ -85,4 +85,4 @@ bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) {
 
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30

mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h

@@ -3,7 +3,7 @@
 
 #include "mediapipe/framework/port.h"
 
-#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #include <array>
 #include <memory>
@@ -40,6 +40,6 @@ bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context);
 
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #endif  // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_

mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc

@@ -1,6 +1,6 @@
 #include "mediapipe/framework/port.h"
 
-#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 
 #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -46,4 +46,4 @@ TEST(ImageToTensorConverterGlUtilsTest, GlTexParameteriOverrider) {
 }  // namespace
 }  // namespace mediapipe
 
-#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20
+#endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -352,11 +352,12 @@ class MetalProcessor : public ImageToTensorConverter {
                                  const RotatedRect& roi,
                                  const Size& output_dims, float range_min,
                                  float range_max) override {
-    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) {
-      return InvalidArgumentError(
-          absl::StrCat("Only BGRA/RGBA textures are supported, passed "
-                       "format: ",
-                       static_cast<uint32_t>(input.format())));
+    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+      return InvalidArgumentError(absl::StrCat(
+          "Only 4-channel texture input formats are supported, passed format: ",
+          static_cast<uint32_t>(input.format())));
     }
 
     @autoreleasepool {

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc

@@ -35,7 +35,8 @@ namespace {
 
 class OpenCvProcessor : public ImageToTensorConverter {
  public:
-  OpenCvProcessor(BorderMode border_mode) {
+  OpenCvProcessor(BorderMode border_mode, Tensor::ElementType tensor_type)
+      : tensor_type_(tensor_type) {
     switch (border_mode) {
       case BorderMode::kReplicate:
         border_mode_ = cv::BORDER_REPLICATE;
@@ -44,6 +45,19 @@ class OpenCvProcessor : public ImageToTensorConverter {
         border_mode_ = cv::BORDER_CONSTANT;
         break;
     }
+    switch (tensor_type_) {
+      case Tensor::ElementType::kInt8:
+        mat_type_ = CV_8SC3;
+        break;
+      case Tensor::ElementType::kFloat32:
+        mat_type_ = CV_32FC3;
+        break;
+      case Tensor::ElementType::kUInt8:
+        mat_type_ = CV_8UC3;
+        break;
+      default:
+        mat_type_ = -1;
+    }
   }
 
   absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
@@ -56,15 +70,30 @@ class OpenCvProcessor : public ImageToTensorConverter {
           absl::StrCat("Only RGBA/RGB formats are supported, passed format: ",
                        static_cast<uint32_t>(input.image_format())));
     }
-    cv::Mat src = mediapipe::formats::MatView(&input);
+    auto src = mediapipe::formats::MatView(&input);
 
     constexpr int kNumChannels = 3;
-    Tensor tensor(
-        Tensor::ElementType::kFloat32,
-        Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
+    Tensor tensor(tensor_type_, Tensor::Shape{1, output_dims.height,
+                                              output_dims.width, kNumChannels});
     auto buffer_view = tensor.GetCpuWriteView();
-    cv::Mat dst(output_dims.height, output_dims.width, CV_32FC3,
-                buffer_view.buffer<float>());
+    cv::Mat dst;
+    switch (tensor_type_) {
+      case Tensor::ElementType::kInt8:
+        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+                      buffer_view.buffer<int8>());
+        break;
+      case Tensor::ElementType::kFloat32:
+        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+                      buffer_view.buffer<float>());
+        break;
+      case Tensor::ElementType::kUInt8:
+        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+                      buffer_view.buffer<uint8>());
+        break;
+      default:
+        return InvalidArgumentError(
+            absl::StrCat("Unsupported tensor type: ", tensor_type_));
+    }
 
     const cv::RotatedRect rotated_rect(cv::Point2f(roi.center_x, roi.center_y),
                                        cv::Size2f(roi.width, roi.height),
@@ -85,7 +114,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
     cv::Mat projection_matrix =
         cv::getPerspectiveTransform(src_points, dst_points);
     cv::Mat transformed;
-    cv::warpPerspective(src, transformed, projection_matrix,
+    cv::warpPerspective(*src, transformed, projection_matrix,
                         cv::Size(dst_width, dst_height),
                         /*flags=*/cv::INTER_LINEAR,
                         /*borderMode=*/border_mode_);
@@ -102,19 +131,29 @@ class OpenCvProcessor : public ImageToTensorConverter {
         auto transform,
         GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                     range_min, range_max));
-    transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
+    transformed.convertTo(dst, mat_type_, transform.scale, transform.offset);
     return tensor;
   }
 
  private:
   enum cv::BorderTypes border_mode_;
+  Tensor::ElementType tensor_type_;
+  int mat_type_;
 };
 
 }  // namespace
 
 absl::StatusOr<std::unique_ptr<ImageToTensorConverter>> CreateOpenCvConverter(
-    CalculatorContext* cc, BorderMode border_mode) {
-  return absl::make_unique<OpenCvProcessor>(border_mode);
+    CalculatorContext* cc, BorderMode border_mode,
+    Tensor::ElementType tensor_type) {
+  if (tensor_type != Tensor::ElementType::kInt8 &&
+      tensor_type != Tensor::ElementType::kFloat32 &&
+      tensor_type != Tensor::ElementType::kUInt8) {
+    return absl::InvalidArgumentError(absl::StrCat(
+        "Tensor type is currently not supported by OpenCvProcessor, type: ",
+        tensor_type));
+  }
+  return absl::make_unique<OpenCvProcessor>(border_mode, tensor_type);
 }
 
 }  // namespace mediapipe

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h

@@ -25,7 +25,8 @@ namespace mediapipe {
 
 // Creates OpenCV image-to-tensor converter.
 absl::StatusOr<std::unique_ptr<ImageToTensorConverter>> CreateOpenCvConverter(
-    CalculatorContext* cc, BorderMode border_mode);
+    CalculatorContext* cc, BorderMode border_mode,
+    Tensor::ElementType tensor_type);
 
 }  // namespace mediapipe
 

mediapipe/calculators/tensor/inference_calculator.cc

@@ -19,9 +19,11 @@
 #include <string>
 #include <vector>
 
-#include "absl/memory/memory.h"
 #include "absl/strings/string_view.h"
+#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
+#include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/tool/subgraph_expansion.h"
+#include "tensorflow/lite/core/api/op_resolver.h"
 
 namespace mediapipe {
 namespace api2 {
@@ -36,12 +38,24 @@ class InferenceCalculatorSelectorImpl
         Subgraph::GetOptions<mediapipe::InferenceCalculatorOptions>(
             subgraph_node);
     std::vector<absl::string_view> impls;
+
     const bool should_use_gpu =
         !options.has_delegate() ||  // Use GPU delegate if not specified
         (options.has_delegate() && options.delegate().has_gpu());
     if (should_use_gpu) {
+      const auto& api = options.delegate().gpu().api();
+      using Gpu = ::mediapipe::InferenceCalculatorOptions::Delegate::Gpu;
       impls.emplace_back("Metal");
-      impls.emplace_back("Gl");
+      const bool prefer_gl_advanced =
+          options.delegate().gpu().use_advanced_gpu_api() &&
+          (api == Gpu::ANY || api == Gpu::OPENGL || api == Gpu::OPENCL);
+      if (prefer_gl_advanced) {
+        impls.emplace_back("GlAdvanced");
+        impls.emplace_back("Gl");
+      } else {
+        impls.emplace_back("Gl");
+        impls.emplace_back("GlAdvanced");
+      }
     }
     impls.emplace_back("Cpu");
     for (const auto& suffix : impls) {
@@ -66,5 +80,17 @@ absl::StatusOr<Packet<TfLiteModelPtr>> InferenceCalculator::GetModelAsPacket(
                       "Must specify TFLite model as path or loaded model.");
 }
 
+absl::StatusOr<Packet<tflite::OpResolver>>
+InferenceCalculator::GetOpResolverAsPacket(CalculatorContext* cc) {
+  if (kSideInOpResolver(cc).IsConnected()) {
+    return kSideInOpResolver(cc).As<tflite::OpResolver>();
+  } else if (kSideInCustomOpResolver(cc).IsConnected()) {
+    return kSideInCustomOpResolver(cc).As<tflite::OpResolver>();
+  }
+  return PacketAdopting<tflite::OpResolver>(
+      std::make_unique<
+          tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates>());
+}
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/tensor/inference_calculator.h

@@ -27,6 +27,7 @@
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/tflite/tflite_model_loader.h"
+#include "tensorflow/lite/core/api/op_resolver.h"
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
@@ -55,8 +56,11 @@ namespace api2 {
 //  TENSORS - Vector of Tensors
 //
 // Input side packet:
+//  DEPRECATED: Prefer to use the "OP_RESOLVER" input side packet instead.
 //  CUSTOM_OP_RESOLVER (optional) - Use a custom op resolver,
 //                                  instead of the builtin one.
+//  OP_RESOLVER (optional) - Use to provide tflite op resolver
+//                           (tflite::OpResolver)
 //  MODEL (optional) - Use to specify TfLite model
 //                     (std::unique_ptr<tflite::FlatBufferModel,
 //                       std::function<void(tflite::FlatBufferModel*)>>)
@@ -95,17 +99,21 @@ namespace api2 {
 class InferenceCalculator : public NodeIntf {
  public:
   static constexpr Input<std::vector<Tensor>> kInTensors{"TENSORS"};
+  // Deprecated. Prefers to use "OP_RESOLVER" input side packet instead.
+  // TODO: Removes the "CUSTOM_OP_RESOLVER" side input after the
+  // migration.
   static constexpr SideInput<tflite::ops::builtin::BuiltinOpResolver>::Optional
       kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"};
+  static constexpr SideInput<tflite::OpResolver>::Optional kSideInOpResolver{
+      "OP_RESOLVER"};
   static constexpr SideInput<TfLiteModelPtr>::Optional kSideInModel{"MODEL"};
   static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
-  static constexpr SideInput<std::string>::Optional kNnApiDelegateCacheDir{
-      "NNAPI_CACHE_DIR"};
-  static constexpr SideInput<std::string>::Optional kNnApiDelegateModelToken{
-      "NNAPI_MODEL_TOKEN"};
-  MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel,
-                          kOutTensors, kNnApiDelegateCacheDir,
-                          kNnApiDelegateModelToken);
+  static constexpr SideInput<
+      mediapipe::InferenceCalculatorOptions::Delegate>::Optional kDelegate{
+      "DELEGATE"};
+  MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver,
+                          kSideInOpResolver, kSideInModel, kOutTensors,
+                          kDelegate);
 
  protected:
   using TfLiteDelegatePtr =
@@ -113,6 +121,9 @@ class InferenceCalculator : public NodeIntf {
 
   absl::StatusOr<Packet<TfLiteModelPtr>> GetModelAsPacket(
       CalculatorContext* cc);
+
+  absl::StatusOr<Packet<tflite::OpResolver>> GetOpResolverAsPacket(
+      CalculatorContext* cc);
 };
 
 struct InferenceCalculatorSelector : public InferenceCalculator {
@@ -123,6 +134,10 @@ struct InferenceCalculatorGl : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorGl";
 };
 
+struct InferenceCalculatorGlAdvanced : public InferenceCalculator {
+  static constexpr char kCalculatorName[] = "InferenceCalculatorGlAdvanced";
+};
+
 struct InferenceCalculatorMetal : public InferenceCalculator {
   static constexpr char kCalculatorName[] = "InferenceCalculatorMetal";
 };