Project import generated by Copybara.

GitOrigin-RevId: bb059a0721c92e8154d33ce8057b3915a25b3d7d

.bazelrc

@@ -58,6 +58,7 @@ build:android_arm64 --fat_apk_cpu=arm64-v8a
 
 # iOS configs.
 build:ios --apple_platform_type=ios
+build:ios --copt=-fno-aligned-allocation
 
 build:ios_i386 --config=ios
 build:ios_i386 --cpu=ios_i386

.bazelversion

@@ -1 +1 @@
-3.7.2
+4.2.1

CONTRIBUTING.md

@@ -5,7 +5,7 @@
 * Bug fixes
 * Documentation fixes
 
-For new feature additions (e.g., new graphs and calculators), we are currently not planning to accept new feature pull requests into the MediaPipe repository. Instead, we like to get contributors to create their own repositories of the new feature and list it at [Awesome MediaPipe](https://mediapipe.org). This will allow contributors to more quickly get their code out to the community.
+For new feature additions (e.g., new graphs and calculators), we are currently not planning to accept new feature pull requests into the MediaPipe repository. Instead, we like to get contributors to create their own repositories of the new feature and list it at [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe). This will allow contributors to more quickly get their code out to the community.
 
 Before sending your pull requests, make sure you followed this list.
 

Dockerfile

@@ -56,7 +56,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=3.7.2
+ARG BAZEL_VERSION=4.2.1
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

WORKSPACE

@@ -122,16 +122,16 @@ http_archive(
 # ...but the Java download is currently broken, so we use the "source" download.
 http_archive(
     name = "com_google_protobuf_javalite",
-    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
+    sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
+    strip_prefix = "protobuf-3.19.1",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
 )
 
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
+    sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
+    strip_prefix = "protobuf-3.19.1",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"],
     patches = [
         "@//third_party:com_google_protobuf_fixes.diff"
     ],
@@ -154,28 +154,29 @@ http_archive(
     sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91",
 )
 
+# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
 http_archive(
     name = "pybind11",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
-        "https://github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
+        "https://github.com/pybind/pybind11/archive/70a58c577eaf067748c2ec31bfd0b0a614cffba6.zip",
     ],
-    sha256 = "616d1c42e4cf14fa27b2a4ff759d7d7b33006fdc5ad8fd603bb2c22622f27020",
-    strip_prefix = "pybind11-2.7.1",
+    sha256 = "b971842fab1b5b8f3815a2302331782b7d137fef0e06502422bc4bc360f4956c",
+    strip_prefix = "pybind11-70a58c577eaf067748c2ec31bfd0b0a614cffba6",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
+# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
 http_archive(
     name = "ceres_solver",
-    url = "https://github.com/ceres-solver/ceres-solver/archive/2.0.0.zip",
+    url = "https://github.com/ceres-solver/ceres-solver/archive/123fba61cf2611a3c8bddc9d91416db26b10b558.zip",
     patches = [
         "@//third_party:ceres_solver_compatibility_fixes.diff"
     ],
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "ceres-solver-2.0.0",
-    sha256 = "db12d37b4cebb26353ae5b7746c7985e00877baa8e7b12dc4d3a1512252fff3b"
+    strip_prefix = "ceres-solver-123fba61cf2611a3c8bddc9d91416db26b10b558",
+    sha256 = "8b7b16ceb363420e0fd499576daf73fa338adb0b1449f58bea7862766baa1ac7"
 )
 
 http_archive(
@@ -249,21 +250,12 @@ http_archive(
     ],
 )
 
-# You may run setup_android.sh to install Android SDK and NDK.
-android_ndk_repository(
-    name = "androidndk",
-)
-
-android_sdk_repository(
-    name = "androidsdk",
-)
-
 # iOS basic build deps.
 
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42",
-    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz",
+    sha256 = "77e8bf6fda706f420a55874ae6ee4df0c9d95da6c7838228b26910fc82eea5a2",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.32.0/rules_apple.0.32.0.tar.gz",
     patches = [
         # Bypass checking ios unit test runner when building MP ios applications.
         "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
@@ -289,10 +281,9 @@ swift_rules_dependencies()
 
 http_archive(
     name = "build_bazel_apple_support",
-    sha256 = "122ebf7fe7d1c8e938af6aeaee0efe788a3a2449ece5a8d6a428cb18d6f88033",
+    sha256 = "741366f79d900c11e11d8efd6cc6c66a31bfb2451178b58e0b5edc6f1db17b35",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
-        "https://github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
+        "https://github.com/bazelbuild/apple_support/releases/download/0.10.0/apple_support.0.10.0.tar.gz"
     ],
 )
 
@@ -350,6 +341,9 @@ maven_install(
         "com.google.auto.value:auto-value:1.8.1",
         "com.google.auto.value:auto-value-annotations:1.8.1",
         "com.google.code.findbugs:jsr305:latest.release",
+        "com.google.android.datatransport:transport-api:3.0.0",
+        "com.google.android.datatransport:transport-backend-cct:3.1.0",
+        "com.google.android.datatransport:transport-runtime:3.1.0",
         "com.google.flogger:flogger-system-backend:0.6",
         "com.google.flogger:flogger:0.6",
         "com.google.guava:guava:27.0.1-android",
@@ -379,9 +373,9 @@ http_archive(
 )
 
 # Tensorflow repo should always go after the other external dependencies.
-# 2021-07-29
-_TENSORFLOW_GIT_COMMIT = "52a2905cbc21034766c08041933053178c5d10e3"
-_TENSORFLOW_SHA256 = "06d4691bcdb700f3275fa0971a1585221c2b9f3dffe867963be565a6643d7f56"
+# 2021-12-02
+_TENSORFLOW_GIT_COMMIT = "18a1dc0ba806dc023808531f0373d9ec068e64bf"
+_TENSORFLOW_SHA256 = "85b90416f7a11339327777bccd634de00ca0de2cf334f5f0727edcb11ff9289a"
 http_archive(
     name = "org_tensorflow",
     urls = [

docs/getting_started/android.md

@@ -29,8 +29,8 @@ APIs (currently in alpha) that are now available in
 
 *   Install MediaPipe following these [instructions](./install.md).
 *   Setup Java Runtime.
-*   Setup Android SDK release 28.0.3 and above.
-*   Setup Android NDK version between 18 and 21.
+*   Setup Android SDK release 30.0.0 and above.
+*   Setup Android NDK version 18 and above.
 
 MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
 below for Android Studio setup). However, if you prefer using MediaPipe without
@@ -47,6 +47,15 @@ export ANDROID_HOME=<path to the Android SDK>
 export ANDROID_NDK_HOME=<path to the Android NDK>
 ```
 
+and add android_ndk_repository() and android_sdk_repository() rules into the
+[`WORKSPACE`](https://github.com/google/mediapipe/blob/master/WORKSPACE) file as
+the following:
+
+```bash
+$ echo "android_sdk_repository(name = \"androidsdk\")" >> WORKSPACE
+$ echo "android_ndk_repository(name = \"androidndk\")" >> WORKSPACE
+```
+
 In order to use MediaPipe on earlier Android versions, MediaPipe needs to switch
 to a lower Android API level. You can achieve this by specifying `api_level =
 $YOUR_INTENDED_API_LEVEL` in android_ndk_repository() and/or

docs/getting_started/android_archive_library.md

@@ -117,7 +117,7 @@ each project.
         implementation 'com.google.flogger:flogger-system-backend:latest.release'
         implementation 'com.google.code.findbugs:jsr305:latest.release'
         implementation 'com.google.guava:guava:27.0.1-android'
-        implementation 'com.google.protobuf:protobuf-java:3.11.4'
+        implementation 'com.google.protobuf:protobuf-javalite:3.19.1'
         // CameraX core library
         def camerax_version = "1.0.0-beta10"
         implementation "androidx.camera:camera-core:$camerax_version"

docs/getting_started/android_solutions.md

@@ -1,28 +1,30 @@
 ---
 layout: default
-title: Android Solutions
+title: MediaPipe Android Solutions
 parent: MediaPipe on Android
 grand_parent: Getting Started
 nav_order: 2
 ---
 
-# Android Solution APIs
+# MediaPipe Android Solutions
 {: .no_toc }
 
 1. TOC
 {:toc}
 ---
 
-Please follow instructions below to use the MediaPipe Solution APIs in Android
-Studio projects and build the Android example apps in the supported MediaPipe
-[solutions](../solutions/solutions.md).
+MediaPipe Android Solution APIs (currently in alpha) are available in:
 
-## Integrate MediaPipe Android Solutions in Android Studio
+*   [MediaPipe Face Detection](../solutions/face_detection#android-solution-api)
+*   [MediaPipe Face Mesh](../solutions/face_mesh#android-solution-api)
+*   [MediaPipe Hands](../solutions/hands#android-solution-api)
 
-MediaPipe Android Solution APIs (currently in alpha) are now available in
+## Incorporation in Android Studio
+
+Prebuilt packages of Android Solution APIs can be found in
 [Google's Maven Repository](https://maven.google.com/web/index.html?#com.google.mediapipe).
-To incorporate MediaPipe Android Solutions into an Android Studio project, add
-the following into the project's Gradle dependencies:
+To incorporate them into an Android Studio project, add the following into the
+project's Gradle dependencies:
 
 ```
 dependencies {
@@ -34,27 +36,20 @@ dependencies {
     implementation 'com.google.mediapipe:facemesh:latest.release'
     // Optional: MediaPipe Hands Solution.
     implementation 'com.google.mediapipe:hands:latest.release'
-    // MediaPipe deps
-    implementation 'com.google.flogger:flogger:0.6'
-    implementation 'com.google.flogger:flogger-system-backend:0.6'
-    implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-java:3.11.4'
-    // CameraX core library
-    def camerax_version = "1.0.0-beta10"
-    implementation "androidx.camera:camera-core:$camerax_version"
-    implementation "androidx.camera:camera-camera2:$camerax_version"
-    implementation "androidx.camera:camera-lifecycle:$camerax_version"
 }
 ```
 
-See the detailed solution APIs usage examples for different use cases in the
-solution example apps'
-[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
-If the prebuilt maven packages are not sufficient, building the MediaPipe
-Android archive library locally by following these
-[instructions](./android_archive_library.md).
+If you need further customization, instead of using the prebuilt maven packages
+consider building a MediaPipe Android Archive library locally from source by
+following these [instructions](./android_archive_library.md).
 
-## Build solution example apps in Android Studio
+## Building solution example apps
+
+Detailed usage examples of the Android Solution APIs can be found in the
+[source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions)
+of the solution example apps.
+
+To build these apps:
 
 1.  Open Android Studio Arctic Fox on Linux, macOS, or Windows.
 
@@ -75,7 +70,62 @@ Android archive library locally by following these
 
 6.  (Optional) Run solutions on CPU.
 
-    MediaPipe solution example apps run the pipeline and the model inference on
-    GPU by default. If needed, for example to run the apps on Android Emulator,
-    set the `RUN_ON_GPU` boolean variable to `false` in the app's
-    MainActivity.java to run the pipeline and the model inference on CPU.
+    MediaPipe solution example apps run the pipeline and model inference on GPU
+    by default. If needed, for example to run the apps on Android Emulator, set
+    the `RUN_ON_GPU` boolean variable to `false` in the app's
+    `MainActivity.java` to run the pipeline and model inference on CPU.
+
+## MediaPipe Solution APIs Terms of Service
+
+Last modified: November 12, 2021
+
+Use of MediaPipe Solution APIs is subject to the
+[Google APIs Terms of Service](https://developers.google.com/terms),
+[Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy),
+and the terms below. Please check back from time to time as these terms and
+policies are occasionally updated.
+
+**Privacy**
+
+When you use MediaPipe Solution APIs, processing of the input data (e.g. images,
+video, text) fully happens on-device, and **MediaPipe does not send that input
+data to Google servers**. As a result, you can use our APIs for processing data
+that should not leave the device.
+
+MediaPipe Android Solution APIs will contact Google servers from time to time in
+order to receive things like bug fixes, updated models, and hardware accelerator
+compatibility information. MediaPipe Android Solution APIs also send metrics
+about the performance and utilization of the APIs in your app to Google. Google
+uses this metrics data to measure performance, API usage, debug, maintain and
+improve the APIs, and detect misuse or abuse, as further described in our
+[Privacy Policy](https://policies.google.com/privacy).
+
+**You are responsible for obtaining informed consent from your app users about
+Google’s processing of MediaPipe metrics data as required by applicable law.**
+
+Data we collect may include the following, across all MediaPipe Android Solution
+APIs:
+
+-   Device information (such as manufacturer, model, OS version and build) and
+    available ML hardware accelerators (GPU and DSP). Used for diagnostics and
+    usage analytics.
+
+-   App identification information (package name / bundle id, app version). Used
+    for diagnostics and usage analytics.
+
+-   API configuration (such as image format, resolution, and MediaPipe version
+    used). Used for diagnostics and usage analytics.
+
+-   Event type (such as initialize, download model, update, run, and detection).
+    Used for diagnostics and usage analytics.
+
+-   Error codes. Used for diagnostics.
+
+-   Performance metrics. Used for diagnostics.
+
+-   Per-installation identifiers that do not uniquely identify a user or
+    physical device. Used for operation of remote configuration and usage
+    analytics.
+
+-   Network request sender IP addresses. Used for remote configuration
+    diagnostics. Collected IP addresses are retained temporarily.

docs/getting_started/install.md

@@ -569,7 +569,7 @@ next section.
 
     Option 1. Follow
     [the official Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 3.7.2 or higher.
+    to install Bazel 4.2.1 or higher.
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-bazelisk.html)
@@ -657,7 +657,7 @@ cameras. Alternatively, you use a video file as input.
 
     Note: Windows' and WSL’s adb versions must be the same version, e.g., if WSL
     has ADB 1.0.39, you need to download the corresponding Windows ADB from
-    [here](https://dl.google.com/android/repository/platform-tools_r26.0.1-windows.zip).
+    [here](https://dl.google.com/android/repository/platform-tools_r30.0.3-windows.zip).
 
 3.  Launch WSL.
 
@@ -796,7 +796,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
     ```bash
     $ docker run -it --name mediapipe mediapipe:latest
 
-    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazelisk run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
+    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world
 
     # Should print:
     # Hello World!

docs/getting_started/python_framework.md

@@ -249,12 +249,12 @@ three stages: initialization and setup, graph run, and graph shutdown.
     graph.start_run()
 
     graph.add_packet_to_input_stream(
-        'in_stream', mp.packet_creator.create_str('abc').at(0))
+        'in_stream', mp.packet_creator.create_string('abc').at(0))
 
     rgb_img = cv2.cvtColor(cv2.imread('/path/to/your/image.png'), cv2.COLOR_BGR2RGB)
     graph.add_packet_to_input_stream(
         'in_stream',
-        mp.packet_creator.create_image_frame(format=mp.ImageFormat.SRGB,
+        mp.packet_creator.create_image_frame(image_format=mp.ImageFormat.SRGB,
                                              data=rgb_img).at(1))
     ```
 

docs/getting_started/troubleshooting.md

@@ -108,14 +108,14 @@ ERROR: No matching distribution found for mediapipe
 
 after running `pip install mediapipe` usually indicates that there is no qualified MediaPipe Python for your system.
 Please note that MediaPipe Python PyPI officially supports the **64-bit**
-version of Python 3.7 and above on the following OS:
+version of Python 3.7 to 3.10 on the following OS:
 
 -   x86_64 Linux
 -   x86_64 macOS 10.15+
 -   amd64 Windows
 
 If the OS is currently supported and you still see this error, please make sure
-that both the Python and pip binary are for Python 3.7 and above. Otherwise,
+that both the Python and pip binary are for Python 3.7 to 3.10. Otherwise,
 please consider building the MediaPipe Python package locally by following the
 instructions [here](python.md#building-mediapipe-python-package).
 

docs/solutions/face_detection.md

@@ -218,14 +218,13 @@ camera.start();
 ### Android Solution API
 
 Please first follow general
-[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Face Detection Solution API
-in the companion
-[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection)
-following
-[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection),
 and learn more in the usage example below.
 
+Supported configuration options:
+
 *   [staticImageMode](#static_image_mode)
 *   [modelSelection](#model_selection)
 
@@ -257,8 +256,15 @@ glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 faceDetection.setResultListener(
     faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
       RelativeKeypoint noseTip =
-          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
       Log.i(
           TAG,
           String.format(
@@ -297,10 +303,17 @@ FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
 FaceDetectionResultImageView imageView = new FaceDetectionResultImageView(this);
 faceDetection.setResultListener(
     faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
       int width = faceDetectionResult.inputBitmap().getWidth();
       int height = faceDetectionResult.inputBitmap().getHeight();
       RelativeKeypoint noseTip =
-          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
       Log.i(
           TAG,
           String.format(
@@ -334,9 +347,9 @@ ActivityResultLauncher<Intent> imageGetter =
             }
           }
         });
-Intent gallery = new Intent(
-    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-imageGetter.launch(gallery);
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
 ```
 
 #### Video Input
@@ -368,8 +381,15 @@ glSurfaceView.setRenderInputImage(true);
 
 faceDetection.setResultListener(
     faceDetectionResult -> {
+      if (faceDetectionResult.multiFaceDetections().isEmpty()) {
+        return;
+      }
       RelativeKeypoint noseTip =
-        FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+          faceDetectionResult
+              .multiFaceDetections()
+              .get(0)
+              .getLocationData()
+              .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
       Log.i(
           TAG,
           String.format(
@@ -398,9 +418,9 @@ ActivityResultLauncher<Intent> videoGetter =
             }
           }
         });
-Intent gallery =
-    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-videoGetter.launch(gallery);
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
 ```
 
 ## Example Apps

docs/solutions/face_mesh.md

@@ -487,12 +487,9 @@ camera.start();
 ### Android Solution API
 
 Please first follow general
-[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Face Mesh Solution API in the
-companion
-[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
-following
-[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh),
 and learn more in the usage example below.
 
 Supported configuration options:
@@ -612,9 +609,9 @@ ActivityResultLauncher<Intent> imageGetter =
             }
           }
         });
-Intent gallery = new Intent(
-    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-imageGetter.launch(gallery);
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
 ```
 
 #### Video Input
@@ -678,9 +675,9 @@ ActivityResultLauncher<Intent> videoGetter =
             }
           }
         });
-Intent gallery =
-    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-videoGetter.launch(gallery);
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
 ```
 
 ## Example Apps

docs/solutions/hands.md

@@ -91,8 +91,10 @@ To detect initial hand locations, we designed a
 mobile real-time uses in a manner similar to the face detection model in
 [MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
 task: our
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite)
-has to work across a variety of hand sizes with a large scale span (~20x)
+[lite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite)
+and
+[full model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite)
+have to work across a variety of hand sizes with a large scale span (~20x)
 relative to the image frame and be able to detect occluded and self-occluded
 hands. Whereas faces have high contrast patterns, e.g., in the eye and mouth
 region, the lack of such features in hands makes it comparatively difficult to
@@ -120,7 +122,7 @@ just 86.22%.
 ### Hand Landmark Model
 
 After the palm detection over the whole image our subsequent hand landmark
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite)
+[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite)
 performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
 the detected hand regions via regression, that is direct coordinate prediction.
 The model learns a consistent internal hand pose representation and is robust
@@ -163,6 +165,11 @@ unrelated, images. Default to `false`.
 
 Maximum number of hands to detect. Default to `2`.
 
+#### model_complexity
+
+Complexity of the hand landmark model: `0` or `1`. Landmark accuracy as well as
+inference latency generally go up with the model complexity. Default to `1`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the hand detection model for the
@@ -190,6 +197,13 @@ of 21 hand landmarks and each landmark is composed of `x`, `y` and `z`. `x` and
 and the smaller the value the closer the landmark is to the camera. The
 magnitude of `z` uses roughly the same scale as `x`.
 
+#### multi_hand_world_landmarks
+
+Collection of detected/tracked hands, where each hand is represented as a list
+of 21 hand landmarks in world coordinates. Each landmark is composed of `x`, `y`
+and `z`: real-world 3D coordinates in meters with the origin at the hand's
+approximate geometric center.
+
 #### multi_handedness
 
 Collection of handedness of the detected/tracked hands (i.e. is it a left or
@@ -212,6 +226,7 @@ Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
 *   [max_num_hands](#max_num_hands)
+*   [model_complexity](#model_complexity)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -256,10 +271,17 @@ with mp_hands.Hands(
           mp_drawing_styles.get_default_hand_connections_style())
     cv2.imwrite(
         '/tmp/annotated_image' + str(idx) + '.png', cv2.flip(annotated_image, 1))
+    # Draw hand world landmarks.
+    if not results.multi_hand_world_landmarks:
+      continue
+    for hand_world_landmarks in results.multi_hand_world_landmarks:
+      mp_drawing.plot_landmarks(
+        hand_world_landmarks, mp_hands.HAND_CONNECTIONS, azimuth=5)
 
 # For webcam input:
 cap = cv2.VideoCapture(0)
 with mp_hands.Hands(
+    model_complexity=0,
     min_detection_confidence=0.5,
     min_tracking_confidence=0.5) as hands:
   while cap.isOpened():
@@ -302,6 +324,7 @@ and a [fun application], and the following usage example.
 Supported configuration options:
 
 *   [maxNumHands](#max_num_hands)
+*   [modelComplexity](#model_complexity)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -351,6 +374,7 @@ const hands = new Hands({locateFile: (file) => {
 }});
 hands.setOptions({
   maxNumHands: 2,
+  modelComplexity: 1,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });
@@ -370,13 +394,10 @@ camera.start();
 ### Android Solution API
 
 Please first follow general
-[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Hands Solution API in the
-companion
-[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
-following
-[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
-and learn more in usage example below.
+[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle
+dependencies and try the Android Solution API in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands),
+and learn more in the usage example below.
 
 Supported configuration options:
 
@@ -391,7 +412,7 @@ Supported configuration options:
 HandsOptions handsOptions =
     HandsOptions.builder()
         .setStaticImageMode(false)
-        .setMaxNumHands(1)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
@@ -414,8 +435,11 @@ glSurfaceView.setRenderInputImage(true);
 
 hands.setResultListener(
     handsResult -> {
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -444,7 +468,7 @@ glSurfaceView.post(
 HandsOptions handsOptions =
     HandsOptions.builder()
         .setStaticImageMode(true)
-        .setMaxNumHands(1)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 
@@ -455,10 +479,13 @@ Hands hands = new Hands(this, handsOptions);
 HandsResultImageView imageView = new HandsResultImageView(this);
 hands.setResultListener(
     handsResult -> {
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
       int width = handsResult.inputBitmap().getWidth();
       int height = handsResult.inputBitmap().getHeight();
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -492,9 +519,9 @@ ActivityResultLauncher<Intent> imageGetter =
             }
           }
         });
-Intent gallery = new Intent(
-    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-imageGetter.launch(gallery);
+Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+imageGetter.launch(pickImageIntent);
 ```
 
 #### Video Input
@@ -504,7 +531,7 @@ imageGetter.launch(gallery);
 HandsOptions handsOptions =
     HandsOptions.builder()
         .setStaticImageMode(false)
-        .setMaxNumHands(1)
+        .setMaxNumHands(2)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
@@ -527,8 +554,11 @@ glSurfaceView.setRenderInputImage(true);
 
 hands.setResultListener(
     handsResult -> {
-      NormalizedLandmark wristLandmark = Hands.getHandLandmark(
-          handsResult, 0, HandLandmark.WRIST);
+      if (result.multiHandLandmarks().isEmpty()) {
+        return;
+      }
+      NormalizedLandmark wristLandmark =
+          handsResult.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
       Log.i(
           TAG,
           String.format(
@@ -557,9 +587,9 @@ ActivityResultLauncher<Intent> videoGetter =
             }
           }
         });
-Intent gallery =
-    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-videoGetter.launch(gallery);
+Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+videoGetter.launch(pickVideoIntent);
 ```
 
 ## Example Apps

docs/solutions/holistic.md

@@ -159,6 +159,11 @@ images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
 is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
 `true`.
 
+#### refine_face_landmarks
+
+Whether to further refine the landmark coordinates around the eyes and lips, and
+output additional landmarks around the irises. Default to `false`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
@@ -241,6 +246,7 @@ Supported configuration options:
 *   [smooth_landmarks](#smooth_landmarks)
 *   [enable_segmentation](#enable_segmentation)
 *   [smooth_segmentation](#smooth_segmentation)
+*   [refine_face_landmarks](#refine_face_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -256,7 +262,8 @@ IMAGE_FILES = []
 with mp_holistic.Holistic(
     static_image_mode=True,
     model_complexity=2,
-    enable_segmentation=True) as holistic:
+    enable_segmentation=True,
+    refine_face_landmarks=True) as holistic:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -350,6 +357,7 @@ Supported configuration options:
 *   [smoothLandmarks](#smooth_landmarks)
 *   [enableSegmentation](#enable_segmentation)
 *   [smoothSegmentation](#smooth_segmentation)
+*   [refineFaceLandmarks](#refineFaceLandmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -421,6 +429,7 @@ holistic.setOptions({
   smoothLandmarks: true,
   enableSegmentation: true,
   smoothSegmentation: true,
+  refineFaceLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });

docs/solutions/models.md

@@ -55,13 +55,14 @@ one over the other.
 ### [Hands](https://google.github.io/mediapipe/solutions/hands)
 
 *   Palm detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection.tflite),
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite),
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
 *   Hand landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark.tflite),
-    [TFLite model (sparse)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_sparse.tflite),
+    [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_lite.tflite),
+    [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-*   [Model card](https://mediapipe.page.link/handmc), [Model card (sparse)](https://mediapipe.page.link/handmc-sparse)
+*   [Model card](https://mediapipe.page.link/handmc)
 
 ### [Pose](https://google.github.io/mediapipe/solutions/pose)
 

docs/solutions/pose.md

@@ -125,7 +125,7 @@ hip midpoints.
 :----------------------------------------------------------------------------------------------------: |
 *Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
 
-### Pose Landmark Model (BlazePose GHUM 3D)
+### Pose Landmark Model (BlazePose [GHUM](https://github.com/google-research/google-research/tree/master/ghum) 3D)
 
 The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
 (see figure below).
@@ -486,6 +486,7 @@ on how to build MediaPipe examples.
     [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
     ([presentation](https://youtu.be/YPpUOTRn5tA))
 *   [Models and model cards](./models.md#pose)
+*   [GHUM & GHUML: Generative 3D Human Shape and Articulated Pose Models](https://github.com/google-research/google-research/tree/master/ghum)
 *   [Web demo](https://code.mediapipe.dev/codepen/pose)
 *   [Python Colab](https://mediapipe.page.link/pose_py_colab)
 

mediapipe/calculators/core/BUILD

@@ -531,9 +531,13 @@ cc_test(
         ":split_vector_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/types:optional",
     ],
 )
 
@@ -1238,7 +1242,6 @@ cc_test(
         "//mediapipe/framework:calculator_profile_cc_proto",
         "//mediapipe/framework:test_calculators",
         "//mediapipe/framework/deps:clock",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:core_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -47,4 +47,8 @@ typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
     BeginLoopMatrixVectorCalculator;
 REGISTER_CALCULATOR(BeginLoopMatrixVectorCalculator);
 
+// A calculator to process std::vector<uint64_t>.
+typedef BeginLoopCalculator<std::vector<uint64_t>> BeginLoopUint64tCalculator;
+REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/graph_profile_calculator_test.cc

@@ -23,7 +23,6 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_profile.pb.h"
 #include "mediapipe/framework/deps/clock.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"

mediapipe/calculators/core/mux_calculator_test.cc

@@ -14,7 +14,11 @@
 
 #include <memory>
 
+#include "absl/status/status.h"
+#include "absl/types/optional.h"
 #include "mediapipe/calculators/core/split_vector_calculator.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -301,4 +305,99 @@ TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
 }
 
 }  // namespace
+
+class PassThroughAndTsBoundUpdateNode : public mediapipe::api2::Node {
+ public:
+  static constexpr mediapipe::api2::Input<int> kInValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<int> kOutValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<int> kOutTsBoundUpdate{
+      "TS_BOUND_UPDATE"};
+  MEDIAPIPE_NODE_CONTRACT(kInValue, kOutValue, kOutTsBoundUpdate);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    kOutValue(cc).Send(kInValue(cc));
+    kOutTsBoundUpdate(cc).SetNextTimestampBound(
+        cc->InputTimestamp().NextAllowedInStream());
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(PassThroughAndTsBoundUpdateNode);
+
+class ToOptionalNode : public mediapipe::api2::Node {
+ public:
+  static constexpr mediapipe::api2::Input<int> kTick{"TICK"};
+  static constexpr mediapipe::api2::Input<int> kInValue{"VALUE"};
+  static constexpr mediapipe::api2::Output<absl::optional<int>> kOutValue{
+      "OUTPUT"};
+  MEDIAPIPE_NODE_CONTRACT(kTick, kInValue, kOutValue);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (kInValue(cc).IsEmpty()) {
+      kOutValue(cc).Send(absl::nullopt);
+    } else {
+      kOutValue(cc).Send({kInValue(cc).Get()});
+    }
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(ToOptionalNode);
+
+namespace {
+
+TEST(MuxCalculatorTest, HandleTimestampBoundUpdates) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "select"
+            node {
+              calculator: "PassThroughAndTsBoundUpdateNode"
+              input_stream: "VALUE:select"
+              output_stream: "VALUE:select_ps"
+              output_stream: "TS_BOUND_UPDATE:ts_bound_update"
+            }
+            node {
+              calculator: "MuxCalculator"
+              input_stream: "INPUT:0:select_ps"
+              input_stream: "INPUT:1:ts_bound_update"
+              input_stream: "SELECT:select"
+              output_stream: "OUTPUT:select_or_ts_bound_update"
+            }
+            node {
+              calculator: "ToOptionalNode"
+              input_stream: "TICK:select"
+              input_stream: "VALUE:select_or_ts_bound_update"
+              output_stream: "OUTPUT:output"
+            }
+          )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto send_value_fn = [&](int value, Timestamp ts) -> absl::Status {
+    MP_RETURN_IF_ERROR(
+        graph.AddPacketToInputStream("select", MakePacket<int>(value).At(ts)));
+    return graph.WaitUntilIdle();
+  };
+
+  MP_ASSERT_OK(send_value_fn(0, Timestamp(1)));
+  ASSERT_EQ(output_packets.size(), 1);
+  EXPECT_EQ(output_packets[0].Get<absl::optional<int>>(), 0);
+
+  MP_ASSERT_OK(send_value_fn(1, Timestamp(2)));
+  ASSERT_EQ(output_packets.size(), 2);
+  EXPECT_EQ(output_packets[1].Get<absl::optional<int>>(), absl::nullopt);
+
+  MP_ASSERT_OK(send_value_fn(0, Timestamp(3)));
+  ASSERT_EQ(output_packets.size(), 3);
+  EXPECT_EQ(output_packets[2].Get<absl::optional<int>>(), 0);
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+}  // namespace
+
 }  // namespace mediapipe

mediapipe/calculators/core/packet_cloner_calculator.cc

@@ -60,7 +60,10 @@ class PacketClonerCalculator : public CalculatorBase {
     const auto calculator_options =
         cc->Options<mediapipe::PacketClonerCalculatorOptions>();
     output_only_when_all_inputs_received_ =
-        calculator_options.output_only_when_all_inputs_received();
+        calculator_options.output_only_when_all_inputs_received() ||
+        calculator_options.output_packets_only_when_all_inputs_received();
+    output_empty_packets_before_all_inputs_received_ =
+        calculator_options.output_packets_only_when_all_inputs_received();
 
     // Parse input streams.
     tick_signal_index_ = cc->Inputs().NumEntries() - 1;
@@ -88,6 +91,9 @@ class PacketClonerCalculator : public CalculatorBase {
         // Return if one of the input is null.
         for (int i = 0; i < tick_signal_index_; ++i) {
           if (current_[i].IsEmpty()) {
+            if (output_empty_packets_before_all_inputs_received_) {
+              SetAllNextTimestampBounds(cc);
+            }
             return absl::OkStatus();
           }
         }
@@ -107,9 +113,17 @@ class PacketClonerCalculator : public CalculatorBase {
   }
 
  private:
+  void SetAllNextTimestampBounds(CalculatorContext* cc) {
+    for (int j = 0; j < tick_signal_index_; ++j) {
+      cc->Outputs().Index(j).SetNextTimestampBound(
+          cc->InputTimestamp().NextAllowedInStream());
+    }
+  }
+
   std::vector<Packet> current_;
   int tick_signal_index_;
   bool output_only_when_all_inputs_received_;
+  bool output_empty_packets_before_all_inputs_received_;
 };
 
 REGISTER_CALCULATOR(PacketClonerCalculator);

mediapipe/calculators/core/packet_cloner_calculator.proto

@@ -28,4 +28,9 @@ message PacketClonerCalculatorOptions {
   // When true, this calculator will drop received TICK packets if any input
   // stream hasn't received a packet yet.
   optional bool output_only_when_all_inputs_received = 1 [default = false];
+
+  // Similar with above, but also transmit empty packet for all streams before
+  // all inputs are received.
+  optional bool output_packets_only_when_all_inputs_received = 2
+      [default = false];
 }

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -45,6 +45,9 @@ namespace mediapipe {
 #if !MEDIAPIPE_DISABLE_GPU
 
 #endif  // !MEDIAPIPE_DISABLE_GPU
+#if defined(MEDIAPIPE_IOS)
+
+#endif  // defined(MEDIAPIPE_IOS)
 
 namespace {
 constexpr char kImageFrameTag[] = "IMAGE";

mediapipe/calculators/tensor/inference_calculator.proto

@@ -34,7 +34,6 @@ option java_outer_classname = "InferenceCalculatorProto";
 //     }
 //   }
 // }
-//
 message InferenceCalculatorOptions {
   extend mediapipe.CalculatorOptions {
     optional InferenceCalculatorOptions ext = 336783863;
@@ -69,8 +68,30 @@ message InferenceCalculatorOptions {
       // Load pre-compiled serialized binary cache to accelerate init process.
       // Only available for OpenCL delegate on Android.
       // Kernel caching will only be enabled if this path is set.
+      //
+      // NOTE: binary cache usage may be skipped if valid serialized model,
+      // specified by "serialized_model_dir", exists.
+      //
+      // TODO: update to cached_kernel_dir
       optional string cached_kernel_path = 2;
 
+      // A dir to load from and save to a pre-compiled serialized model used to
+      // accelerate init process.
+      //
+      // NOTE: available for OpenCL delegate on Android only when
+      // "use_advanced_gpu_api" is set to true and "model_token" is set
+      // properly.
+      //
+      // NOTE: serialized model takes precedence over binary cache
+      // specified by "cached_kernel_path", which still can be used if
+      // serialized model is invalid or missing.
+      optional string serialized_model_dir = 7;
+
+      // Unique token identifying the model. Used in conjunction with
+      // "serialized_model_dir". It is the caller's responsibility to ensure
+      // there is no clash of the tokens.
+      optional string model_token = 8;
+
       // Encapsulated compilation/runtime tradeoffs.
       enum InferenceUsage {
         UNSPECIFIED = 0;

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -20,6 +20,7 @@
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/util/tflite/config.h"
 
 #if MEDIAPIPE_TFLITE_GL_INFERENCE
@@ -49,8 +50,8 @@ class InferenceCalculatorGlImpl
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  absl::Status ReadKernelsFromFile();
-  absl::Status WriteKernelsToFile();
+  absl::Status ReadGpuCaches();
+  absl::Status SaveGpuCaches();
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
   absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
@@ -82,6 +83,8 @@ class InferenceCalculatorGlImpl
 
   bool use_kernel_caching_ = false;
   std::string cached_kernel_filename_;
+  bool use_serialized_model_ = false;
+  std::string serialized_model_path_;
 };
 
 absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
@@ -114,6 +117,9 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
   tflite_gpu_runner_usage_ = delegate.gpu().usage();
   use_kernel_caching_ =
       use_advanced_gpu_api_ && delegate.gpu().has_cached_kernel_path();
+  use_serialized_model_ = use_advanced_gpu_api_ &&
+                          delegate.gpu().has_serialized_model_dir() &&
+                          delegate.gpu().has_model_token();
   use_gpu_delegate_ = !use_advanced_gpu_api_;
 
   if (use_kernel_caching_) {
@@ -123,6 +129,12 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
                               ".ker";
 #endif  // MEDIAPIPE_ANDROID
   }
+  if (use_serialized_model_) {
+#ifdef MEDIAPIPE_ANDROID
+    serialized_model_path_ = mediapipe::file::JoinPath(
+        delegate.gpu().serialized_model_dir(), delegate.gpu().model_token());
+#endif  // MEDIAPIPE_ANDROID
+  }
 
   // When use_advanced_gpu_api_, model loading is handled in InitTFLiteGPURunner
   // for everything.
@@ -210,7 +222,7 @@ absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
-absl::Status InferenceCalculatorGlImpl::WriteKernelsToFile() {
+absl::Status InferenceCalculatorGlImpl::SaveGpuCaches() {
 #ifdef MEDIAPIPE_ANDROID
   if (use_kernel_caching_) {
     // Save kernel file.
@@ -220,12 +232,22 @@ absl::Status InferenceCalculatorGlImpl::WriteKernelsToFile() {
     MP_RETURN_IF_ERROR(
         mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
   }
+  if (use_serialized_model_) {
+    // Save serialized model file.
+    ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
+                     tflite_gpu_runner_->GetSerializedModel());
+    absl::string_view serialized_model(
+        reinterpret_cast<char*>(serialized_model_vec.data()),
+        serialized_model_vec.size());
+    MP_RETURN_IF_ERROR(
+        mediapipe::file::SetContents(serialized_model_path_, serialized_model));
+  }
 #endif  // MEDIAPIPE_ANDROID
   return absl::OkStatus();
 }
 
 absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
-  MP_RETURN_IF_ERROR(WriteKernelsToFile());
+  MP_RETURN_IF_ERROR(SaveGpuCaches());
   if (use_gpu_delegate_) {
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
       gpu_buffers_in_.clear();
@@ -239,17 +261,24 @@ absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
-absl::Status InferenceCalculatorGlImpl::ReadKernelsFromFile() {
+absl::Status InferenceCalculatorGlImpl::ReadGpuCaches() {
 #ifdef MEDIAPIPE_ANDROID
-  if (use_kernel_caching_) {
+  if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
     // Load pre-compiled kernel file.
-    if (mediapipe::File::Exists(cached_kernel_filename_)) {
-      std::string cache_str;
-      MP_RETURN_IF_ERROR(
-          mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
-      std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
-      tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
-    }
+    std::string cache_str;
+    MP_RETURN_IF_ERROR(
+        mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
+    std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
+    tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
+  }
+  if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
+    // Load serialized model file.
+    std::string serialized_model_str;
+    MP_RETURN_IF_ERROR(
+        file::GetContents(serialized_model_path_, &serialized_model_str));
+    std::vector<uint8_t> serialized_model_vec(serialized_model_str.begin(),
+                                              serialized_model_str.end());
+    tflite_gpu_runner_->SetSerializedModel(std::move(serialized_model_vec));
   }
 #endif  // MEDIAPIPE_ANDROID
   return absl::OkStatus();
@@ -313,7 +342,7 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner(
                          tflite_gpu_runner_->GetOutputShapes()[i].c};
   }
 
-  MP_RETURN_IF_ERROR(ReadKernelsFromFile());
+  MP_RETURN_IF_ERROR(ReadGpuCaches());
 
   MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
 

mediapipe/calculators/tensor/inference_calculator_test.cc

@@ -39,8 +39,6 @@
 
 namespace mediapipe {
 
-using ::tflite::Interpreter;
-
 void DoSmokeTest(const std::string& graph_proto) {
   const int width = 8;
   const int height = 8;

mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc

@@ -26,7 +26,6 @@
 #include "tensorflow/core/framework/types.h"
 
 namespace mediapipe {
-
 namespace {
 
 constexpr char kTransposeOptionsString[] =

mediapipe/calculators/tflite/ssd_anchors_calculator.proto

@@ -24,20 +24,20 @@ message SsdAnchorsCalculatorOptions {
     optional SsdAnchorsCalculatorOptions ext = 247258239;
   }
   // Size of input images.
-  required int32 input_size_width = 1;
-  required int32 input_size_height = 2;
+  optional int32 input_size_width = 1;   // required
+  optional int32 input_size_height = 2;  // required
 
   // Min and max scales for generating anchor boxes on feature maps.
-  required float min_scale = 3;
-  required float max_scale = 4;
+  optional float min_scale = 3;  // required
+  optional float max_scale = 4;  // required
 
   // The offset for the center of anchors. The value is in the scale of stride.
   // E.g. 0.5 meaning 0.5 * |current_stride| in pixels.
-  required float anchor_offset_x = 5 [default = 0.5];
-  required float anchor_offset_y = 6 [default = 0.5];
+  optional float anchor_offset_x = 5 [default = 0.5];  // required
+  optional float anchor_offset_y = 6 [default = 0.5];  // required
 
   // Number of output feature maps to generate the anchors on.
-  required int32 num_layers = 7;
+  optional int32 num_layers = 7;  // required
   // Sizes of output feature maps to create anchors. Either feature_map size or
   // stride should be provided.
   repeated int32 feature_map_width = 8;

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto

@@ -26,12 +26,12 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
   }
 
   // The number of output classes predicted by the detection model.
-  required int32 num_classes = 1;
+  optional int32 num_classes = 1;  // required
   // The number of output boxes predicted by the detection model.
-  required int32 num_boxes = 2;
+  optional int32 num_boxes = 2;  // required
   // The number of output values per boxes predicted by the detection model. The
   // values contain bounding boxes, keypoints, etc.
-  required int32 num_coords = 3;
+  optional int32 num_coords = 3;  // required
 
   // The offset of keypoint coordinates in the location tensor.
   optional int32 keypoint_coord_offset = 9;

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.proto

@@ -31,7 +31,7 @@ message TfLiteTensorsToLandmarksCalculatorOptions {
   }
 
   // Number of landmarks from the output of the model.
-  required int32 num_landmarks = 1;
+  optional int32 num_landmarks = 1;  // required
 
   // Size of the input image for the model. These options are used only when
   // normalized landmarks are needed. Z coordinate is scaled as X assuming

mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.proto

@@ -24,9 +24,9 @@ message TfLiteTensorsToSegmentationCalculatorOptions {
   }
 
   // Dimensions of input segmentation tensor to process.
-  required int32 tensor_width = 1;
-  required int32 tensor_height = 2;
-  required int32 tensor_channels = 3;
+  optional int32 tensor_width = 1;     // required
+  optional int32 tensor_height = 2;    // required
+  optional int32 tensor_channels = 3;  // required
 
   // How much to use previous mask when computing current one; range [0-1].
   // This is a tradeoff between responsiveness (0.0) and accuracy (1.0).

mediapipe/calculators/util/BUILD

@@ -176,7 +176,6 @@ cc_test(
         ":filter_detections_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
@@ -215,7 +214,6 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:timestamp",
         "//mediapipe/framework/deps:clock",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
@@ -488,7 +486,6 @@ cc_test(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
@@ -772,7 +769,6 @@ cc_test(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",
         "//mediapipe/framework/port:gtest_main",
@@ -869,7 +865,6 @@ cc_test(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:gtest_main",
@@ -1067,7 +1062,6 @@ cc_test(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
@@ -1169,6 +1163,7 @@ cc_library(
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/port:rectangle",
         "//mediapipe/framework/port:status",
+        "//mediapipe/util:rectangle_util",
         "@com_google_absl//absl/memory",
     ],
     alwayslink = 1,
@@ -1216,7 +1211,6 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework:packet",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
@@ -1284,6 +1278,7 @@ cc_library(
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image",
         "//mediapipe/framework/port:logging",
@@ -1343,7 +1338,6 @@ cc_test(
         ":detection_classifications_merger_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/port:gtest_main",

mediapipe/calculators/util/association_calculator.h

@@ -26,20 +26,10 @@
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/rectangle.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/rectangle_util.h"
 
 namespace mediapipe {
 
-// Computes the overlap similarity based on Intersection over Union (IoU) of
-// two rectangles.
-inline float OverlapSimilarity(const Rectangle_f& rect1,
-                               const Rectangle_f& rect2) {
-  if (!rect1.Intersects(rect2)) return 0.0f;
-  // Compute IoU similarity score.
-  const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area();
-  const float normalization = rect1.Area() + rect2.Area() - intersection_area;
-  return normalization > 0.0f ? intersection_area / normalization : 0.0f;
-}
-
 // AssocationCalculator<T> accepts multiple inputs of vectors of type T that can
 // be converted to Rectangle_f. The output is a vector of type T that contains
 // elements from the input vectors that don't overlap with each other. When
@@ -187,7 +177,7 @@ class AssociationCalculator : public CalculatorBase {
 
     for (auto uit = current->begin(); uit != current->end();) {
       ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit));
-      if (OverlapSimilarity(cur_rect, prev_rect) >
+      if (CalculateIou(cur_rect, prev_rect) >
           options_.min_similarity_threshold()) {
         std::pair<bool, int> prev_id = GetId(*uit);
         // If prev_id.first is false when some element doesn't have an ID,
@@ -232,7 +222,7 @@ class AssociationCalculator : public CalculatorBase {
         }
         const Rectangle_f& prev_rect = get_prev_rectangle.value();
 
-        if (OverlapSimilarity(cur_rect, prev_rect) >
+        if (CalculateIou(cur_rect, prev_rect) >
             options_.min_similarity_threshold()) {
           std::pair<bool, int> prev_id = GetId(prev_input_vec[ui]);
           // If prev_id.first is false when some element doesn't have an ID,

mediapipe/calculators/util/association_calculator_test.cc

@@ -16,7 +16,6 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"

mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc

@@ -14,9 +14,9 @@
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"

mediapipe/calculators/util/detections_to_rects_calculator_test.cc

@@ -19,7 +19,6 @@
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"

mediapipe/calculators/util/detections_to_render_data_calculator_test.cc

@@ -17,7 +17,6 @@
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
 #include "mediapipe/framework/packet.h"
@@ -34,8 +33,6 @@ constexpr char kDetectionsTag[] = "DETECTIONS";
 constexpr char kRenderDataTag[] = "RENDER_DATA";
 constexpr char kDetectionListTag[] = "DETECTION_LIST";
 
-using ::testing::DoubleNear;
-
 // Error tolerance for pixels, distances, etc.
 static constexpr double kErrorTolerance = 1e-5;
 

mediapipe/calculators/util/filter_detections_calculator_test.cc

@@ -16,7 +16,6 @@
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"

mediapipe/calculators/util/inverse_matrix_calculator.cc

@@ -33,9 +33,17 @@ class InverseMatrixCalculatorImpl : public NodeImpl<InverseMatrixCalculator> {
         kInputMatrix(cc).Get().data());
 
     Eigen::Matrix<float, 4, 4, Eigen::RowMajor> inverse_matrix;
-    bool inverse_check;
-    matrix.computeInverseWithCheck(inverse_matrix, inverse_check);
-    RET_CHECK(inverse_check) << "Inverse matrix cannot be calculated.";
+    bool inverse_check = false;
+    // The matrix is invertible if the absolute value of its determinant is
+    // greater than this threshold. Quite small threshold is selected to enable
+    // inverting valid matrices containing relatively small values resulting in
+    // a small determinant.
+    constexpr double kAbsDeterminantThreshold =
+        Eigen::NumTraits<double>::epsilon();
+    matrix.computeInverseWithCheck(inverse_matrix, inverse_check,
+                                   kAbsDeterminantThreshold);
+    RET_CHECK(inverse_check)
+        << "Inverse matrix cannot be calculated for: " << matrix;
 
     std::array<float, 16> output;
     Eigen::Map<Eigen::Matrix<float, 4, 4, Eigen::RowMajor>>(

mediapipe/calculators/util/inverse_matrix_calculator_test.cc

@@ -42,7 +42,11 @@ void RunTest(const std::array<float, 16>& matrix,
 
   const auto& inverse_matrix = output_packets[0].Get<std::array<float, 16>>();
 
-  EXPECT_THAT(inverse_matrix, testing::Eq(expected_inverse_matrix));
+  EXPECT_THAT(
+      inverse_matrix,
+      testing::Pointwise(testing::FloatEq(),
+                         absl::MakeSpan(expected_inverse_matrix.data(),
+                                        expected_inverse_matrix.size())));
 
   // Fully close graph at end, otherwise calculator+tensors are destroyed
   // after calling WaitUntilDone().
@@ -122,5 +126,25 @@ TEST(InverseMatrixCalculatorTest, Rotation90) {
   RunTest(matrix, expected_inverse_matrix);
 }
 
+TEST(InverseMatrixCalculatorTest, CheckPrecision) {
+  // clang-format off
+  std::array<float, 16> matrix = {
+    0.00001f,  0.0f,      0.0f, 0.0f,
+    0.0f,      0.00001f,  0.0f, 0.0f,
+    0.0f,      0.0f,      1.0f, 0.0f,
+    0.0f,      0.0f,      0.0f, 1.0f,
+  };
+
+  std::array<float, 16> expected_inverse_matrix = {
+    100000.0f, 0.0f,      0.0f, 0.0f,
+    0.0f,      100000.0f, 0.0f, 0.0f,
+    0.0f,      0.0f,      1.0f, 0.0f,
+    0.0f,      0.0f,      0.0f, 1.0f,
+  };
+  // clang-format on
+
+  RunTest(matrix, expected_inverse_matrix);
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/util/landmark_projection_calculator_test.cc

@@ -6,9 +6,9 @@
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"

mediapipe/calculators/util/packet_latency_calculator_test.cc

@@ -16,7 +16,6 @@
 #include "mediapipe/calculators/util/latency.pb.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/clock.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"

mediapipe/calculators/util/to_image_calculator.cc

@@ -14,6 +14,7 @@
 
 #include <memory>
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/formats/image.h"
@@ -23,23 +24,23 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 
-#if !MEDIAPIPE_DISABLE_GPU
-#include "mediapipe/gpu/gl_calculator_helper.h"
-#endif  // !MEDIAPIPE_DISABLE_GPU
-
 namespace mediapipe {
+namespace api2 {
 
-namespace {
-constexpr char kImageFrameTag[] = "IMAGE_CPU";
-constexpr char kGpuBufferTag[] = "IMAGE_GPU";
-constexpr char kImageTag[] = "IMAGE";
-}  // namespace
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+class Nothing {};
+using GpuBuffer = Nothing;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
 
 // A calculator for converting from legacy MediaPipe datatypes into a
 // unified image container.
 //
 // Inputs:
 //   One of the following two tags:
+//   IMAGE:  An Image, ImageFrame, or GpuBuffer containing input image.
 //   IMAGE_CPU:  An ImageFrame containing input image.
 //   IMAGE_GPU:  A GpuBuffer containing input image.
 //
@@ -49,107 +50,44 @@ constexpr char kImageTag[] = "IMAGE";
 // Note:
 //   No CPU/GPU conversion is done.
 //
-class ToImageCalculator : public CalculatorBase {
+class ToImageCalculator : public Node {
  public:
   ToImageCalculator() = default;
   ~ToImageCalculator() override = default;
 
-  static absl::Status GetContract(CalculatorContract* cc);
+  static constexpr Input<
+      OneOf<mediapipe::Image, mediapipe::ImageFrame, GpuBuffer>>::Optional kIn{
+      "IMAGE"};
+  static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
+  static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
+  static constexpr Output<mediapipe::Image> kOut{"IMAGE"};
+  MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc);
 
   // From Calculator.
-  absl::Status Open(CalculatorContext* cc) override;
   absl::Status Process(CalculatorContext* cc) override;
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  absl::Status RenderGpu(CalculatorContext* cc);
-  absl::Status RenderCpu(CalculatorContext* cc);
-
-  bool gpu_input_ = false;
-  bool gpu_initialized_ = false;
-#if !MEDIAPIPE_DISABLE_GPU
-  mediapipe::GlCalculatorHelper gpu_helper_;
-#endif  // !MEDIAPIPE_DISABLE_GPU
+  absl::StatusOr<Packet<Image>> GetInputImage(CalculatorContext* cc);
 };
-REGISTER_CALCULATOR(ToImageCalculator);
+MEDIAPIPE_REGISTER_NODE(ToImageCalculator);
 
-absl::Status ToImageCalculator::GetContract(CalculatorContract* cc) {
-  cc->Outputs().Tag(kImageTag).Set<mediapipe::Image>();
-
-  bool gpu_input = false;
-
-  if (cc->Inputs().HasTag(kImageFrameTag) &&
-      cc->Inputs().HasTag(kGpuBufferTag)) {
+absl::Status ToImageCalculator::UpdateContract(CalculatorContract* cc) {
+  int num_inputs = static_cast<int>(kIn(cc).IsConnected()) +
+                   static_cast<int>(kInCpu(cc).IsConnected()) +
+                   static_cast<int>(kInGpu(cc).IsConnected());
+  if (num_inputs != 1) {
     return absl::InternalError("Cannot have multiple inputs.");
   }
 
-  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-#if !MEDIAPIPE_DISABLE_GPU
-    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
-    gpu_input = true;
-#else
-    RET_CHECK_FAIL() << "GPU is disabled. Cannot use IMAGE_GPU stream.";
-#endif  // !MEDIAPIPE_DISABLE_GPU
-  }
-  if (cc->Inputs().HasTag(kImageFrameTag)) {
-    cc->Inputs().Tag(kImageFrameTag).Set<mediapipe::ImageFrame>();
-  }
-
-  if (gpu_input) {
-#if !MEDIAPIPE_DISABLE_GPU
-    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
-#endif  // !MEDIAPIPE_DISABLE_GPU
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status ToImageCalculator::Open(CalculatorContext* cc) {
-  cc->SetOffset(TimestampDiff(0));
-
-  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    gpu_input_ = true;
-  }
-
-  if (gpu_input_) {
-#if !MEDIAPIPE_DISABLE_GPU
-    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#endif
-  }  //  !MEDIAPIPE_DISABLE_GPU
-
   return absl::OkStatus();
 }
 
 absl::Status ToImageCalculator::Process(CalculatorContext* cc) {
-  if (gpu_input_) {
-#if !MEDIAPIPE_DISABLE_GPU
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([&cc]() -> absl::Status {
-      auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
-      // Wrap texture pointer; shallow copy.
-      auto output = std::make_unique<mediapipe::Image>(input);
-      cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp());
-      return absl::OkStatus();
-    }));
-#endif  // !MEDIAPIPE_DISABLE_GPU
-  } else {
-    // The input ImageFrame.
-    auto& input = cc->Inputs().Tag(kImageFrameTag).Get<mediapipe::ImageFrame>();
-    // Make a copy of the input packet to co-own the input ImageFrame.
-    Packet* packet_copy_ptr =
-        new Packet(cc->Inputs().Tag(kImageFrameTag).Value());
-    // Create an output Image that (co-)owns a new ImageFrame that points to
-    // the same pixel data as the input ImageFrame and also owns the packet
-    // copy. As a result, the output Image indirectly co-owns the input
-    // ImageFrame. This ensures a correct life span of the shared pixel data.
-    std::unique_ptr<mediapipe::Image> output =
-        std::make_unique<mediapipe::Image>(
-            std::make_shared<mediapipe::ImageFrame>(
-                input.Format(), input.Width(), input.Height(),
-                input.WidthStep(), const_cast<uint8*>(input.PixelData()),
-                [packet_copy_ptr](uint8*) { delete packet_copy_ptr; }));
-    cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp());
-  }
-
+  ASSIGN_OR_RETURN(auto output, GetInputImage(cc));
+  kOut(cc).Send(output.At(cc->InputTimestamp()));
   return absl::OkStatus();
 }
 
@@ -157,4 +95,43 @@ absl::Status ToImageCalculator::Close(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
+// Wrap ImageFrameSharedPtr; shallow copy.
+absl::StatusOr<Packet<Image>> FromImageFrame(Packet<ImageFrame> packet) {
+  return MakePacket<Image, std::shared_ptr<mediapipe::ImageFrame>>(
+      std::const_pointer_cast<mediapipe::ImageFrame>(
+          SharedPtrWithPacket<mediapipe::ImageFrame>(packet)));
+}
+
+// Wrap texture pointer; shallow copy.
+absl::StatusOr<Packet<Image>> FromGpuBuffer(Packet<GpuBuffer> packet) {
+#if !MEDIAPIPE_DISABLE_GPU
+  const GpuBuffer& buffer = *packet;
+  return MakePacket<Image, const GpuBuffer&>(buffer);
+#else
+  return absl::UnimplementedError("GPU processing is disabled in build flags");
+#endif  // !MEDIAPIPE_DISABLE_GPU
+}
+
+absl::StatusOr<Packet<Image>> ToImageCalculator::GetInputImage(
+    CalculatorContext* cc) {
+  if (kIn(cc).IsConnected()) {
+    return kIn(cc).Visit(
+        [&](const mediapipe::Image&) {
+          return absl::StatusOr<Packet<Image>>(kIn(cc).As<Image>());
+        },
+        [&](const mediapipe::ImageFrame&) {
+          return FromImageFrame(kIn(cc).As<ImageFrame>());
+        },
+        [&](const GpuBuffer&) {
+          return FromGpuBuffer(kIn(cc).As<GpuBuffer>());
+        });
+  } else if (kInCpu(cc).IsConnected()) {
+    return FromImageFrame(kInCpu(cc).As<ImageFrame>());
+  } else if (kInGpu(cc).IsConnected()) {
+    return FromGpuBuffer(kInGpu(cc).As<GpuBuffer>());
+  }
+  return absl::InvalidArgumentError("No input found.");
+}
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/examples/android/solutions/facedetection/build.gradle

@@ -35,17 +35,7 @@ dependencies {
     testImplementation 'junit:junit:4.+'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
-    // MediaPipe Face Detection Solution components.
+    // MediaPipe Face Detection Solution.
     implementation 'com.google.mediapipe:solution-core:latest.release'
     implementation 'com.google.mediapipe:facedetection:latest.release'
-    // MediaPipe deps
-    implementation 'com.google.flogger:flogger:0.6'
-    implementation 'com.google.flogger:flogger-system-backend:0.6'
-    implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-java:3.11.4'
-    // CameraX core library
-    def camerax_version = "1.0.0-beta10"
-    implementation "androidx.camera:camera-core:$camerax_version"
-    implementation "androidx.camera:camera-camera2:$camerax_version"
-    implementation "androidx.camera:camera-lifecycle:$camerax_version"
 }

mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml

@@ -11,6 +11,9 @@
   <!-- For using the camera -->
   <uses-permission android:name="android.permission.CAMERA" />
   <uses-feature android:name="android.hardware.camera" />
+  <!-- For logging solution events -->
+  <uses-permission android:name="android.permission.INTERNET" />
+  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 
   <application
       android:allowBackup="true"

mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java

@@ -32,9 +32,9 @@ public class FaceDetectionResultImageView extends AppCompatImageView {
   private static final String TAG = "FaceDetectionResultImageView";
 
   private static final int KEYPOINT_COLOR = Color.RED;
-  private static final int KEYPOINT_RADIUS = 15;
+  private static final int KEYPOINT_RADIUS = 8; // Pixels
   private static final int BBOX_COLOR = Color.GREEN;
-  private static final int BBOX_THICKNESS = 10;
+  private static final int BBOX_THICKNESS = 5; // Pixels
   private Bitmap latest;
 
   public FaceDetectionResultImageView(Context context) {

mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/MainActivity.java

@@ -28,7 +28,6 @@ import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.exifinterface.media.ExifInterface;
 // ContentResolver dependency
-import com.google.mediapipe.formats.proto.LocationDataProto.LocationData.RelativeKeypoint;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
 import com.google.mediapipe.solutioncore.VideoInput;
@@ -36,6 +35,7 @@ import com.google.mediapipe.solutions.facedetection.FaceDetection;
 import com.google.mediapipe.solutions.facedetection.FaceDetectionOptions;
 import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
 import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
+import com.google.mediapipe.formats.proto.LocationDataProto.LocationData.RelativeKeypoint;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -98,6 +98,43 @@ public class MainActivity extends AppCompatActivity {
     }
   }
 
+  private Bitmap downscaleBitmap(Bitmap originalBitmap) {
+    double aspectRatio = (double) originalBitmap.getWidth() / originalBitmap.getHeight();
+    int width = imageView.getWidth();
+    int height = imageView.getHeight();
+    if (((double) imageView.getWidth() / imageView.getHeight()) > aspectRatio) {
+      width = (int) (height * aspectRatio);
+    } else {
+      height = (int) (width / aspectRatio);
+    }
+    return Bitmap.createScaledBitmap(originalBitmap, width, height, false);
+  }
+
+  private Bitmap rotateBitmap(Bitmap inputBitmap, InputStream imageData) throws IOException {
+    int orientation =
+        new ExifInterface(imageData)
+            .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+    if (orientation == ExifInterface.ORIENTATION_NORMAL) {
+      return inputBitmap;
+    }
+    Matrix matrix = new Matrix();
+    switch (orientation) {
+      case ExifInterface.ORIENTATION_ROTATE_90:
+        matrix.postRotate(90);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_180:
+        matrix.postRotate(180);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_270:
+        matrix.postRotate(270);
+        break;
+      default:
+        matrix.postRotate(0);
+    }
+    return Bitmap.createBitmap(
+        inputBitmap, 0, 0, inputBitmap.getWidth(), inputBitmap.getHeight(), matrix, true);
+  }
+
   /** Sets up the UI components for the static image demo. */
   private void setupStaticImageDemoUiComponents() {
     // The Intent to access gallery and read images as bitmap.
@@ -111,37 +148,16 @@ public class MainActivity extends AppCompatActivity {
                   Bitmap bitmap = null;
                   try {
                     bitmap =
-                        MediaStore.Images.Media.getBitmap(
-                            this.getContentResolver(), resultIntent.getData());
+                        downscaleBitmap(
+                            MediaStore.Images.Media.getBitmap(
+                                this.getContentResolver(), resultIntent.getData()));
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap reading error:" + e);
                   }
                   try {
                     InputStream imageData =
                         this.getContentResolver().openInputStream(resultIntent.getData());
-                    int orientation =
-                        new ExifInterface(imageData)
-                            .getAttributeInt(
-                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
-                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
-                      Matrix matrix = new Matrix();
-                      switch (orientation) {
-                        case ExifInterface.ORIENTATION_ROTATE_90:
-                          matrix.postRotate(90);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_180:
-                          matrix.postRotate(180);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_270:
-                          matrix.postRotate(270);
-                          break;
-                        default:
-                          matrix.postRotate(0);
-                      }
-                      bitmap =
-                          Bitmap.createBitmap(
-                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
-                    }
+                    bitmap = rotateBitmap(bitmap, imageData);
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap rotation error:" + e);
                   }
@@ -159,9 +175,9 @@ public class MainActivity extends AppCompatActivity {
             setupStaticImageModePipeline();
           }
           // Reads images from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-          imageGetter.launch(gallery);
+          Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+          pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+          imageGetter.launch(pickImageIntent);
         });
     imageView = new FaceDetectionResultImageView(this);
   }
@@ -224,9 +240,9 @@ public class MainActivity extends AppCompatActivity {
           stopCurrentPipeline();
           setupStreamingModePipeline(InputSource.VIDEO);
           // Reads video from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-          videoGetter.launch(gallery);
+          Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+          pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+          videoGetter.launch(pickVideoIntent);
         });
   }
 
@@ -318,8 +334,15 @@ public class MainActivity extends AppCompatActivity {
 
   private void logNoseTipKeypoint(
       FaceDetectionResult result, int faceIndex, boolean showPixelValues) {
+    if (result.multiFaceDetections().isEmpty()) {
+      return;
+    }
     RelativeKeypoint noseTip =
-        FaceDetection.getFaceKeypoint(result, faceIndex, FaceKeypoint.NOSE_TIP);
+        result
+            .multiFaceDetections()
+            .get(faceIndex)
+            .getLocationData()
+            .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
     // For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates.
     if (showPixelValues) {
       int width = result.inputBitmap().getWidth();

mediapipe/examples/android/solutions/facemesh/build.gradle

@@ -7,7 +7,7 @@ android {
     buildToolsVersion "30.0.3"
 
     defaultConfig {
-        applicationId "com.google.mediapipe.apps.hands"
+        applicationId "com.google.mediapipe.apps.facemesh"
         minSdkVersion 21
         targetSdkVersion 30
         versionCode 1
@@ -35,17 +35,7 @@ dependencies {
     testImplementation 'junit:junit:4.+'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
-    // MediaPipe Face Mesh Solution components.
+    // MediaPipe Face Mesh Solution.
     implementation 'com.google.mediapipe:solution-core:latest.release'
     implementation 'com.google.mediapipe:facemesh:latest.release'
-    // MediaPipe deps
-    implementation 'com.google.flogger:flogger:0.6'
-    implementation 'com.google.flogger:flogger-system-backend:0.6'
-    implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-java:3.11.4'
-    // CameraX core library
-    def camerax_version = "1.0.0-beta10"
-    implementation "androidx.camera:camera-core:$camerax_version"
-    implementation "androidx.camera:camera-camera2:$camerax_version"
-    implementation "androidx.camera:camera-lifecycle:$camerax_version"
 }

mediapipe/examples/android/solutions/facemesh/src/main/AndroidManifest.xml

@@ -11,11 +11,14 @@
   <!-- For using the camera -->
   <uses-permission android:name="android.permission.CAMERA" />
   <uses-feature android:name="android.hardware.camera" />
+  <!-- For logging solution events -->
+  <uses-permission android:name="android.permission.INTERNET" />
+  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 
   <application
       android:allowBackup="true"
       android:icon="@mipmap/ic_launcher"
-      android:label="MediaPipe FaceMesh"
+      android:label="MediaPipe Face Mesh"
       android:roundIcon="@mipmap/ic_launcher_round"
       android:supportsRtl="true"
       android:theme="@style/AppTheme">

mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java

@@ -34,19 +34,19 @@ public class FaceMeshResultImageView extends AppCompatImageView {
   private static final String TAG = "FaceMeshResultImageView";
 
   private static final int TESSELATION_COLOR = Color.parseColor("#70C0C0C0");
-  private static final int TESSELATION_THICKNESS = 5;
+  private static final int TESSELATION_THICKNESS = 3; // Pixels
   private static final int RIGHT_EYE_COLOR = Color.parseColor("#FF3030");
-  private static final int RIGHT_EYE_THICKNESS = 8;
+  private static final int RIGHT_EYE_THICKNESS = 5; // Pixels
   private static final int RIGHT_EYEBROW_COLOR = Color.parseColor("#FF3030");
-  private static final int RIGHT_EYEBROW_THICKNESS = 8;
+  private static final int RIGHT_EYEBROW_THICKNESS = 5; // Pixels
   private static final int LEFT_EYE_COLOR = Color.parseColor("#30FF30");
-  private static final int LEFT_EYE_THICKNESS = 8;
+  private static final int LEFT_EYE_THICKNESS = 5; // Pixels
   private static final int LEFT_EYEBROW_COLOR = Color.parseColor("#30FF30");
-  private static final int LEFT_EYEBROW_THICKNESS = 8;
+  private static final int LEFT_EYEBROW_THICKNESS = 5; // Pixels
   private static final int FACE_OVAL_COLOR = Color.parseColor("#E0E0E0");
-  private static final int FACE_OVAL_THICKNESS = 8;
+  private static final int FACE_OVAL_THICKNESS = 5; // Pixels
   private static final int LIPS_COLOR = Color.parseColor("#E0E0E0");
-  private static final int LIPS_THICKNESS = 8;
+  private static final int LIPS_THICKNESS = 5; // Pixels
   private Bitmap latest;
 
   public FaceMeshResultImageView(Context context) {

mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/MainActivity.java

@@ -99,6 +99,43 @@ public class MainActivity extends AppCompatActivity {
     }
   }
 
+  private Bitmap downscaleBitmap(Bitmap originalBitmap) {
+    double aspectRatio = (double) originalBitmap.getWidth() / originalBitmap.getHeight();
+    int width = imageView.getWidth();
+    int height = imageView.getHeight();
+    if (((double) imageView.getWidth() / imageView.getHeight()) > aspectRatio) {
+      width = (int) (height * aspectRatio);
+    } else {
+      height = (int) (width / aspectRatio);
+    }
+    return Bitmap.createScaledBitmap(originalBitmap, width, height, false);
+  }
+
+  private Bitmap rotateBitmap(Bitmap inputBitmap, InputStream imageData) throws IOException {
+    int orientation =
+        new ExifInterface(imageData)
+            .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+    if (orientation == ExifInterface.ORIENTATION_NORMAL) {
+      return inputBitmap;
+    }
+    Matrix matrix = new Matrix();
+    switch (orientation) {
+      case ExifInterface.ORIENTATION_ROTATE_90:
+        matrix.postRotate(90);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_180:
+        matrix.postRotate(180);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_270:
+        matrix.postRotate(270);
+        break;
+      default:
+        matrix.postRotate(0);
+    }
+    return Bitmap.createBitmap(
+        inputBitmap, 0, 0, inputBitmap.getWidth(), inputBitmap.getHeight(), matrix, true);
+  }
+
   /** Sets up the UI components for the static image demo. */
   private void setupStaticImageDemoUiComponents() {
     // The Intent to access gallery and read images as bitmap.
@@ -112,37 +149,16 @@ public class MainActivity extends AppCompatActivity {
                   Bitmap bitmap = null;
                   try {
                     bitmap =
-                        MediaStore.Images.Media.getBitmap(
-                            this.getContentResolver(), resultIntent.getData());
+                        downscaleBitmap(
+                            MediaStore.Images.Media.getBitmap(
+                                this.getContentResolver(), resultIntent.getData()));
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap reading error:" + e);
                   }
                   try {
                     InputStream imageData =
                         this.getContentResolver().openInputStream(resultIntent.getData());
-                    int orientation =
-                        new ExifInterface(imageData)
-                            .getAttributeInt(
-                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
-                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
-                      Matrix matrix = new Matrix();
-                      switch (orientation) {
-                        case ExifInterface.ORIENTATION_ROTATE_90:
-                          matrix.postRotate(90);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_180:
-                          matrix.postRotate(180);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_270:
-                          matrix.postRotate(270);
-                          break;
-                        default:
-                          matrix.postRotate(0);
-                      }
-                      bitmap =
-                          Bitmap.createBitmap(
-                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
-                    }
+                    bitmap = rotateBitmap(bitmap, imageData);
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap rotation error:" + e);
                   }
@@ -160,9 +176,9 @@ public class MainActivity extends AppCompatActivity {
             setupStaticImageModePipeline();
           }
           // Reads images from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-          imageGetter.launch(gallery);
+          Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+          pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+          imageGetter.launch(pickImageIntent);
         });
     imageView = new FaceMeshResultImageView(this);
   }
@@ -224,9 +240,9 @@ public class MainActivity extends AppCompatActivity {
           stopCurrentPipeline();
           setupStreamingModePipeline(InputSource.VIDEO);
           // Reads video from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-          videoGetter.launch(gallery);
+          Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+          pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+          videoGetter.launch(pickVideoIntent);
         });
   }
 

mediapipe/examples/android/solutions/hands/build.gradle

@@ -35,17 +35,7 @@ dependencies {
     testImplementation 'junit:junit:4.+'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
-    // MediaPipe Hands Solution components.
+    // MediaPipe Hands Solution.
     implementation 'com.google.mediapipe:solution-core:latest.release'
     implementation 'com.google.mediapipe:hands:latest.release'
-    // MediaPipe deps
-    implementation 'com.google.flogger:flogger:0.6'
-    implementation 'com.google.flogger:flogger-system-backend:0.6'
-    implementation 'com.google.guava:guava:27.0.1-android'
-    implementation 'com.google.protobuf:protobuf-java:3.11.4'
-    // CameraX core library
-    def camerax_version = "1.0.0-beta10"
-    implementation "androidx.camera:camera-core:$camerax_version"
-    implementation "androidx.camera:camera-camera2:$camerax_version"
-    implementation "androidx.camera:camera-lifecycle:$camerax_version"
 }

mediapipe/examples/android/solutions/hands/src/main/AndroidManifest.xml

@@ -11,6 +11,9 @@
   <!-- For using the camera -->
   <uses-permission android:name="android.permission.CAMERA" />
   <uses-feature android:name="android.hardware.camera" />
+  <!-- For logging solution events -->
+  <uses-permission android:name="android.permission.INTERNET" />
+  <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 
   <application
       android:allowBackup="true"

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java

@@ -28,7 +28,16 @@ import java.util.List;
 public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
   private static final String TAG = "HandsResultGlRenderer";
 
-  private static final float CONNECTION_THICKNESS = 20.0f;
+  private static final float[] LEFT_HAND_CONNECTION_COLOR = new float[] {0.2f, 1f, 0.2f, 1f};
+  private static final float[] RIGHT_HAND_CONNECTION_COLOR = new float[] {1f, 0.2f, 0.2f, 1f};
+  private static final float CONNECTION_THICKNESS = 25.0f;
+  private static final float[] LEFT_HAND_HOLLOW_CIRCLE_COLOR = new float[] {0.2f, 1f, 0.2f, 1f};
+  private static final float[] RIGHT_HAND_HOLLOW_CIRCLE_COLOR = new float[] {1f, 0.2f, 0.2f, 1f};
+  private static final float HOLLOW_CIRCLE_RADIUS = 0.01f;
+  private static final float[] LEFT_HAND_LANDMARK_COLOR = new float[] {1f, 0.2f, 0.2f, 1f};
+  private static final float[] RIGHT_HAND_LANDMARK_COLOR = new float[] {0.2f, 1f, 0.2f, 1f};
+  private static final float LANDMARK_RADIUS = 0.008f;
+  private static final int NUM_SEGMENTS = 120;
   private static final String VERTEX_SHADER =
       "uniform mat4 uProjectionMatrix;\n"
           + "attribute vec4 vPosition;\n"
@@ -37,12 +46,14 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
           + "}";
   private static final String FRAGMENT_SHADER =
       "precision mediump float;\n"
+          + "uniform vec4 uColor;\n"
           + "void main() {\n"
-          + "  gl_FragColor = vec4(0, 1, 0, 1);\n"
+          + "  gl_FragColor = uColor;\n"
           + "}";
   private int program;
   private int positionHandle;
   private int projectionMatrixHandle;
+  private int colorHandle;
 
   private int loadShader(int type, String shaderCode) {
     int shader = GLES20.glCreateShader(type);
@@ -61,6 +72,7 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
     GLES20.glLinkProgram(program);
     positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
     projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
+    colorHandle = GLES20.glGetUniformLocation(program, "uColor");
   }
 
   @Override
@@ -74,7 +86,22 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
 
     int numHands = result.multiHandLandmarks().size();
     for (int i = 0; i < numHands; ++i) {
-      drawLandmarks(result.multiHandLandmarks().get(i).getLandmarkList());
+      boolean isLeftHand = result.multiHandedness().get(i).getLabel().equals("Left");
+      drawConnections(
+          result.multiHandLandmarks().get(i).getLandmarkList(),
+          isLeftHand ? LEFT_HAND_CONNECTION_COLOR : RIGHT_HAND_CONNECTION_COLOR);
+      for (NormalizedLandmark landmark : result.multiHandLandmarks().get(i).getLandmarkList()) {
+        // Draws the landmark.
+        drawCircle(
+            landmark.getX(),
+            landmark.getY(),
+            isLeftHand ? LEFT_HAND_LANDMARK_COLOR : RIGHT_HAND_LANDMARK_COLOR);
+        // Draws a hollow circle around the landmark.
+        drawHollowCircle(
+            landmark.getX(),
+            landmark.getY(),
+            isLeftHand ? LEFT_HAND_HOLLOW_CIRCLE_COLOR : RIGHT_HAND_HOLLOW_CIRCLE_COLOR);
+      }
     }
   }
 
@@ -87,7 +114,8 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
     GLES20.glDeleteProgram(program);
   }
 
-  private void drawLandmarks(List<NormalizedLandmark> handLandmarkList) {
+  private void drawConnections(List<NormalizedLandmark> handLandmarkList, float[] colorArray) {
+    GLES20.glUniform4fv(colorHandle, 1, colorArray, 0);
     for (Hands.Connection c : Hands.HAND_CONNECTIONS) {
       NormalizedLandmark start = handLandmarkList.get(c.start());
       NormalizedLandmark end = handLandmarkList.get(c.end());
@@ -103,4 +131,51 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
       GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
     }
   }
+
+  private void drawCircle(float x, float y, float[] colorArray) {
+    GLES20.glUniform4fv(colorHandle, 1, colorArray, 0);
+    int vertexCount = NUM_SEGMENTS + 2;
+    float[] vertices = new float[vertexCount * 3];
+    vertices[0] = x;
+    vertices[1] = y;
+    vertices[2] = 0;
+    for (int i = 1; i < vertexCount; i++) {
+      float angle = 2.0f * i * (float) Math.PI / NUM_SEGMENTS;
+      int currentIndex = 3 * i;
+      vertices[currentIndex] = x + (float) (LANDMARK_RADIUS * Math.cos(angle));
+      vertices[currentIndex + 1] = y + (float) (LANDMARK_RADIUS * Math.sin(angle));
+      vertices[currentIndex + 2] = 0;
+    }
+    FloatBuffer vertexBuffer =
+        ByteBuffer.allocateDirect(vertices.length * 4)
+            .order(ByteOrder.nativeOrder())
+            .asFloatBuffer()
+            .put(vertices);
+    vertexBuffer.position(0);
+    GLES20.glEnableVertexAttribArray(positionHandle);
+    GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 0, vertexBuffer);
+    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_FAN, 0, vertexCount);
+  }
+
+  private void drawHollowCircle(float x, float y, float[] colorArray) {
+    GLES20.glUniform4fv(colorHandle, 1, colorArray, 0);
+    int vertexCount = NUM_SEGMENTS + 1;
+    float[] vertices = new float[vertexCount * 3];
+    for (int i = 0; i < vertexCount; i++) {
+      float angle = 2.0f * i * (float) Math.PI / NUM_SEGMENTS;
+      int currentIndex = 3 * i;
+      vertices[currentIndex] = x + (float) (HOLLOW_CIRCLE_RADIUS * Math.cos(angle));
+      vertices[currentIndex + 1] = y + (float) (HOLLOW_CIRCLE_RADIUS * Math.sin(angle));
+      vertices[currentIndex + 2] = 0;
+    }
+    FloatBuffer vertexBuffer =
+        ByteBuffer.allocateDirect(vertices.length * 4)
+            .order(ByteOrder.nativeOrder())
+            .asFloatBuffer()
+            .put(vertices);
+    vertexBuffer.position(0);
+    GLES20.glEnableVertexAttribArray(positionHandle);
+    GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false, 0, vertexBuffer);
+    GLES20.glDrawArrays(GLES20.GL_LINE_STRIP, 0, vertexCount);
+  }
 }

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java

@@ -31,10 +31,15 @@ import java.util.List;
 public class HandsResultImageView extends AppCompatImageView {
   private static final String TAG = "HandsResultImageView";
 
-  private static final int LANDMARK_COLOR = Color.RED;
-  private static final int LANDMARK_RADIUS = 15;
-  private static final int CONNECTION_COLOR = Color.GREEN;
-  private static final int CONNECTION_THICKNESS = 10;
+  private static final int LEFT_HAND_CONNECTION_COLOR = Color.parseColor("#30FF30");
+  private static final int RIGHT_HAND_CONNECTION_COLOR = Color.parseColor("#FF3030");
+  private static final int CONNECTION_THICKNESS = 8; // Pixels
+  private static final int LEFT_HAND_HOLLOW_CIRCLE_COLOR = Color.parseColor("#30FF30");
+  private static final int RIGHT_HAND_HOLLOW_CIRCLE_COLOR = Color.parseColor("#FF3030");
+  private static final int HOLLOW_CIRCLE_WIDTH = 5; // Pixels
+  private static final int LEFT_HAND_LANDMARK_COLOR = Color.parseColor("#FF3030");
+  private static final int RIGHT_HAND_LANDMARK_COLOR = Color.parseColor("#30FF30");
+  private static final int LANDMARK_RADIUS = 10; // Pixels
   private Bitmap latest;
 
   public HandsResultImageView(Context context) {
@@ -62,7 +67,11 @@ public class HandsResultImageView extends AppCompatImageView {
     int numHands = result.multiHandLandmarks().size();
     for (int i = 0; i < numHands; ++i) {
       drawLandmarksOnCanvas(
-          result.multiHandLandmarks().get(i).getLandmarkList(), canvas, width, height);
+          result.multiHandLandmarks().get(i).getLandmarkList(),
+          result.multiHandedness().get(i).getLabel().equals("Left"),
+          canvas,
+          width,
+          height);
     }
   }
 
@@ -75,11 +84,16 @@ public class HandsResultImageView extends AppCompatImageView {
   }
 
   private void drawLandmarksOnCanvas(
-      List<NormalizedLandmark> handLandmarkList, Canvas canvas, int width, int height) {
+      List<NormalizedLandmark> handLandmarkList,
+      boolean isLeftHand,
+      Canvas canvas,
+      int width,
+      int height) {
     // Draw connections.
     for (Hands.Connection c : Hands.HAND_CONNECTIONS) {
       Paint connectionPaint = new Paint();
-      connectionPaint.setColor(CONNECTION_COLOR);
+      connectionPaint.setColor(
+          isLeftHand ? LEFT_HAND_CONNECTION_COLOR : RIGHT_HAND_CONNECTION_COLOR);
       connectionPaint.setStrokeWidth(CONNECTION_THICKNESS);
       NormalizedLandmark start = handLandmarkList.get(c.start());
       NormalizedLandmark end = handLandmarkList.get(c.end());
@@ -91,11 +105,23 @@ public class HandsResultImageView extends AppCompatImageView {
           connectionPaint);
     }
     Paint landmarkPaint = new Paint();
-    landmarkPaint.setColor(LANDMARK_COLOR);
-    // Draw landmarks.
+    landmarkPaint.setColor(isLeftHand ? LEFT_HAND_LANDMARK_COLOR : RIGHT_HAND_LANDMARK_COLOR);
+    // Draws landmarks.
     for (LandmarkProto.NormalizedLandmark landmark : handLandmarkList) {
       canvas.drawCircle(
           landmark.getX() * width, landmark.getY() * height, LANDMARK_RADIUS, landmarkPaint);
     }
+    // Draws hollow circles around landmarks.
+    landmarkPaint.setColor(
+        isLeftHand ? LEFT_HAND_HOLLOW_CIRCLE_COLOR : RIGHT_HAND_HOLLOW_CIRCLE_COLOR);
+    landmarkPaint.setStrokeWidth(HOLLOW_CIRCLE_WIDTH);
+    landmarkPaint.setStyle(Paint.Style.STROKE);
+    for (LandmarkProto.NormalizedLandmark landmark : handLandmarkList) {
+      canvas.drawCircle(
+          landmark.getX() * width,
+          landmark.getY() * height,
+          LANDMARK_RADIUS + HOLLOW_CIRCLE_WIDTH,
+          landmarkPaint);
+    }
   }
 }

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java

@@ -28,6 +28,7 @@ import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.exifinterface.media.ExifInterface;
 // ContentResolver dependency
+import com.google.mediapipe.formats.proto.LandmarkProto.Landmark;
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@@ -100,6 +101,43 @@ public class MainActivity extends AppCompatActivity {
     }
   }
 
+  private Bitmap downscaleBitmap(Bitmap originalBitmap) {
+    double aspectRatio = (double) originalBitmap.getWidth() / originalBitmap.getHeight();
+    int width = imageView.getWidth();
+    int height = imageView.getHeight();
+    if (((double) imageView.getWidth() / imageView.getHeight()) > aspectRatio) {
+      width = (int) (height * aspectRatio);
+    } else {
+      height = (int) (width / aspectRatio);
+    }
+    return Bitmap.createScaledBitmap(originalBitmap, width, height, false);
+  }
+
+  private Bitmap rotateBitmap(Bitmap inputBitmap, InputStream imageData) throws IOException {
+    int orientation =
+        new ExifInterface(imageData)
+            .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+    if (orientation == ExifInterface.ORIENTATION_NORMAL) {
+      return inputBitmap;
+    }
+    Matrix matrix = new Matrix();
+    switch (orientation) {
+      case ExifInterface.ORIENTATION_ROTATE_90:
+        matrix.postRotate(90);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_180:
+        matrix.postRotate(180);
+        break;
+      case ExifInterface.ORIENTATION_ROTATE_270:
+        matrix.postRotate(270);
+        break;
+      default:
+        matrix.postRotate(0);
+    }
+    return Bitmap.createBitmap(
+        inputBitmap, 0, 0, inputBitmap.getWidth(), inputBitmap.getHeight(), matrix, true);
+  }
+
   /** Sets up the UI components for the static image demo. */
   private void setupStaticImageDemoUiComponents() {
     // The Intent to access gallery and read images as bitmap.
@@ -113,37 +151,16 @@ public class MainActivity extends AppCompatActivity {
                   Bitmap bitmap = null;
                   try {
                     bitmap =
-                        MediaStore.Images.Media.getBitmap(
-                            this.getContentResolver(), resultIntent.getData());
+                        downscaleBitmap(
+                            MediaStore.Images.Media.getBitmap(
+                                this.getContentResolver(), resultIntent.getData()));
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap reading error:" + e);
                   }
                   try {
                     InputStream imageData =
                         this.getContentResolver().openInputStream(resultIntent.getData());
-                    int orientation =
-                        new ExifInterface(imageData)
-                            .getAttributeInt(
-                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
-                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
-                      Matrix matrix = new Matrix();
-                      switch (orientation) {
-                        case ExifInterface.ORIENTATION_ROTATE_90:
-                          matrix.postRotate(90);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_180:
-                          matrix.postRotate(180);
-                          break;
-                        case ExifInterface.ORIENTATION_ROTATE_270:
-                          matrix.postRotate(270);
-                          break;
-                        default:
-                          matrix.postRotate(0);
-                      }
-                      bitmap =
-                          Bitmap.createBitmap(
-                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
-                    }
+                    bitmap = rotateBitmap(bitmap, imageData);
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap rotation error:" + e);
                   }
@@ -161,9 +178,9 @@ public class MainActivity extends AppCompatActivity {
             setupStaticImageModePipeline();
           }
           // Reads images from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
-          imageGetter.launch(gallery);
+          Intent pickImageIntent = new Intent(Intent.ACTION_PICK);
+          pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*");
+          imageGetter.launch(pickImageIntent);
         });
     imageView = new HandsResultImageView(this);
   }
@@ -177,7 +194,7 @@ public class MainActivity extends AppCompatActivity {
             this,
             HandsOptions.builder()
                 .setStaticImageMode(true)
-                .setMaxNumHands(1)
+                .setMaxNumHands(2)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
 
@@ -225,9 +242,9 @@ public class MainActivity extends AppCompatActivity {
           stopCurrentPipeline();
           setupStreamingModePipeline(InputSource.VIDEO);
           // Reads video from gallery.
-          Intent gallery =
-              new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
-          videoGetter.launch(gallery);
+          Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+          pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+          videoGetter.launch(pickVideoIntent);
         });
   }
 
@@ -253,7 +270,7 @@ public class MainActivity extends AppCompatActivity {
             this,
             HandsOptions.builder()
                 .setStaticImageMode(false)
-                .setMaxNumHands(1)
+                .setMaxNumHands(2)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
     hands.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
@@ -320,7 +337,11 @@ public class MainActivity extends AppCompatActivity {
   }
 
   private void logWristLandmark(HandsResult result, boolean showPixelValues) {
-    NormalizedLandmark wristLandmark = Hands.getHandLandmark(result, 0, HandLandmark.WRIST);
+    if (result.multiHandLandmarks().isEmpty()) {
+      return;
+    }
+    NormalizedLandmark wristLandmark =
+        result.multiHandLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
     // For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates.
     if (showPixelValues) {
       int width = result.inputBitmap().getWidth();
@@ -337,5 +358,16 @@ public class MainActivity extends AppCompatActivity {
               "MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               wristLandmark.getX(), wristLandmark.getY()));
     }
+    if (result.multiHandWorldLandmarks().isEmpty()) {
+      return;
+    }
+    Landmark wristWorldLandmark =
+        result.multiHandWorldLandmarks().get(0).getLandmarkList().get(HandLandmark.WRIST);
+    Log.i(
+        TAG,
+        String.format(
+            "MediaPipe Hand wrist world coordinates (in meters with the origin at the hand's"
+                + " approximate geometric center): x=%f m, y=%f m, z=%f m",
+            wristWorldLandmark.getX(), wristWorldLandmark.getY(), wristWorldLandmark.getZ()));
   }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD

@@ -37,7 +37,7 @@ android_binary(
     srcs = glob(["*.java"]),
     assets = [
         "//mediapipe/graphs/hand_tracking:hand_detection_mobile_gpu.binarypb",
-        "//mediapipe/modules/palm_detection:palm_detection.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
     ],
     assets_dir = "",
     manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -37,9 +37,11 @@ android_binary(
     srcs = glob(["*.java"]),
     assets = [
         "//mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu.binarypb",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_lite.tflite",
         "//mediapipe/modules/hand_landmark:handedness.txt",
-        "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
-        "//mediapipe/modules/palm_detection:palm_detection.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_lite.tflite",
     ],
     assets_dir = "",
     manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
@@ -53,6 +55,7 @@ android_binary(
         "outputVideoStreamName": "output_video",
         "flipFramesVertically": "True",
         "converterNumBuffers": "2",
+        # "modelComplexity": "0" # 0=lite, 1=heavy, not specified=heavy
     },
     multidex = "native",
     deps = [

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java

@@ -14,6 +14,9 @@
 
 package com.google.mediapipe.apps.handtrackinggpu;
 
+import android.content.pm.ApplicationInfo;
+import android.content.pm.PackageManager;
+import android.content.pm.PackageManager.NameNotFoundException;
 import android.os.Bundle;
 import android.util.Log;
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
@@ -30,6 +33,7 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
   private static final String TAG = "MainActivity";
 
   private static final String INPUT_NUM_HANDS_SIDE_PACKET_NAME = "num_hands";
+  private static final String INPUT_MODEL_COMPLEXITY = "model_complexity";
   private static final String OUTPUT_LANDMARKS_STREAM_NAME = "hand_landmarks";
   // Max number of hands to detect/process.
   private static final int NUM_HANDS = 2;
@@ -38,9 +42,22 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
   protected void onCreate(Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
 
+    ApplicationInfo applicationInfo;
+    try {
+      applicationInfo =
+          getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
+    } catch (NameNotFoundException e) {
+      throw new AssertionError(e);
+    }
+
     AndroidPacketCreator packetCreator = processor.getPacketCreator();
     Map<String, Packet> inputSidePackets = new HashMap<>();
     inputSidePackets.put(INPUT_NUM_HANDS_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_HANDS));
+    if (applicationInfo.metaData.containsKey("modelComplexity")) {
+      inputSidePackets.put(
+          INPUT_MODEL_COMPLEXITY,
+          packetCreator.createInt32(applicationInfo.metaData.getInt("modelComplexity")));
+    }
     processor.setInputSidePackets(inputSidePackets);
 
     // To show verbose logging, run:

mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD

@@ -39,7 +39,7 @@ android_binary(
         "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb",
         "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
         "//mediapipe/modules/face_landmark:face_landmark.tflite",
-        "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
         "//mediapipe/modules/hand_landmark:handedness.txt",
         "//mediapipe/modules/holistic_landmark:hand_recrop.tflite",
         "//mediapipe/modules/pose_detection:pose_detection.tflite",

mediapipe/examples/desktop/autoflip/calculators/BUILD

@@ -74,6 +74,7 @@ cc_library(
         ":content_zooming_calculator_state",
         "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:location_data_cc_proto",
@@ -290,6 +291,7 @@ cc_library(
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/container:btree",
     ],
     alwayslink = 1,
 )
@@ -389,6 +391,7 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/container:btree",
         "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/strings",
     ],

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc

@@ -22,6 +22,7 @@
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_builder.h"
@@ -57,6 +58,8 @@ constexpr float kFieldOfView = 60;
 constexpr char kStateCache[] = "STATE_CACHE";
 // Tolerance for zooming out recentering.
 constexpr float kPixelTolerance = 3;
+// Returns 'true' when camera is moving (pan/tilt/zoom) & 'false' for no motion.
+constexpr char kCameraActive[] = "CAMERA_ACTIVE";
 
 namespace mediapipe {
 namespace autoflip {
@@ -181,6 +184,9 @@ absl::Status ContentZoomingCalculator::GetContract(
   if (cc->InputSidePackets().HasTag(kStateCache)) {
     cc->InputSidePackets().Tag(kStateCache).Set<StateCacheType*>();
   }
+  if (cc->Outputs().HasTag(kCameraActive)) {
+    cc->Outputs().Tag(kCameraActive).Set<bool>();
+  }
   return absl::OkStatus();
 }
 
@@ -649,6 +655,16 @@ absl::Status ContentZoomingCalculator::Process(
     path_solver_tilt_->ClearHistory();
     path_solver_zoom_->ClearHistory();
   }
+  const bool camera_active =
+      is_animating || pan_state || tilt_state || zoom_state;
+  // Waiting for first rect before setting any value of the camera active flag
+  // so we avoid setting it to false during initialization.
+  if (cc->Outputs().HasTag(kCameraActive) &&
+      first_rect_timestamp_ != Timestamp::Unset()) {
+    cc->Outputs()
+        .Tag(kCameraActive)
+        .AddPacket(MakePacket<bool>(camera_active).At(cc->InputTimestamp()));
+  }
 
   // Compute smoothed zoom camera path.
   MP_RETURN_IF_ERROR(path_solver_zoom_->AddObservation(

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/container/btree_set.h"
 #include "absl/flags/flag.h"
 #include "absl/strings/string_view.h"
 #include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h"
@@ -54,7 +55,8 @@ const char kConfig[] = R"(
 const int kTestFrameWidth = 640;
 const int kTestFrameHeight = 480;
 
-void AddFrames(const int number_of_frames, const std::set<int>& skip_frames,
+void AddFrames(const int number_of_frames,
+               const absl::btree_set<int>& skip_frames,
                CalculatorRunner* runner) {
   cv::Mat image =
       cv::imread(file::JoinPath("./",
@@ -78,7 +80,8 @@ void AddFrames(const int number_of_frames, const std::set<int>& skip_frames,
   }
 }
 
-void CheckOutput(const int number_of_frames, const std::set<int>& shot_frames,
+void CheckOutput(const int number_of_frames,
+                 const absl::btree_set<int>& shot_frames,
                  const std::vector<Packet>& output_packets) {
   ASSERT_EQ(number_of_frames, output_packets.size());
   for (int i = 0; i < number_of_frames; i++) {

mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc

@@ -18,6 +18,7 @@
 #include <utility>
 #include <vector>
 
+#include "absl/container/btree_map.h"
 #include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h"
 #include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -178,8 +179,8 @@ absl::Status SignalFusingCalculator::Close(mediapipe::CalculatorContext* cc) {
 
 absl::Status SignalFusingCalculator::ProcessScene(
     mediapipe::CalculatorContext* cc) {
-  std::map<std::string, int> detection_count;
-  std::map<std::string, float> multiframe_score;
+  absl::btree_map<std::string, int> detection_count;
+  absl::btree_map<std::string, float> multiframe_score;
   // Create a unified score for all items with temporal ids.
   for (const Frame& frame : scene_frames_) {
     for (const auto& detection : frame.input_detections) {

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc

@@ -282,8 +282,12 @@ absl::Status KinematicPathSolver::UpdatePixelsPerDegree(
 
 absl::Status KinematicPathSolver::UpdateMinMaxLocation(const int min_location,
                                                        const int max_location) {
-  RET_CHECK(initialized_)
-      << "UpdateMinMaxLocation called before first observation added.";
+  if (!initialized_) {
+    max_location_ = max_location;
+    min_location_ = min_location;
+    return absl::OkStatus();
+  }
+
   double prior_distance = max_location_ - min_location_;
   double updated_distance = max_location - min_location;
   double scale_change = updated_distance / prior_distance;

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc

@@ -435,6 +435,23 @@ TEST(KinematicPathSolverTest, PassBorderTest) {
   EXPECT_FLOAT_EQ(state, 404.56668);
 }
 
+TEST(KinematicPathSolverTest, PassUpdateUpdateMinMaxLocationIfUninitialized) {
+  KinematicOptions options;
+  options.set_min_motion_to_reframe(2.0);
+  options.set_max_velocity(1000);
+  KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
+  MP_EXPECT_OK(solver.UpdateMinMaxLocation(0, 500));
+}
+
+TEST(KinematicPathSolverTest, PassUpdateUpdateMinMaxLocationIfInitialized) {
+  KinematicOptions options;
+  options.set_min_motion_to_reframe(2.0);
+  options.set_max_velocity(1000);
+  KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
+  MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
+  MP_EXPECT_OK(solver.UpdateMinMaxLocation(0, 500));
+}
+
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/ios/handdetectiongpu/BUILD

@@ -55,7 +55,7 @@ objc_library(
     name = "HandDetectionGpuAppLibrary",
     data = [
         "//mediapipe/graphs/hand_tracking:hand_detection_mobile_gpu_binary_graph",
-        "//mediapipe/modules/palm_detection:palm_detection.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
     ],
     deps = [
         "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary",

mediapipe/examples/ios/handtrackinggpu/BUILD

@@ -62,9 +62,9 @@ objc_library(
     copts = ["-std=c++17"],
     data = [
         "//mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu.binarypb",
-        "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
         "//mediapipe/modules/hand_landmark:handedness.txt",
-        "//mediapipe/modules/palm_detection:palm_detection.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
     ],
     deps = [
         "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary",

mediapipe/examples/ios/holistictrackinggpu/BUILD

@@ -57,7 +57,7 @@ objc_library(
         "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb",
         "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
         "//mediapipe/modules/face_landmark:face_landmark.tflite",
-        "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
         "//mediapipe/modules/hand_landmark:handedness.txt",
         "//mediapipe/modules/holistic_landmark:hand_recrop.tflite",
         "//mediapipe/modules/pose_detection:pose_detection.tflite",

mediapipe/framework/BUILD

@@ -1187,7 +1187,6 @@ cc_test(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/api2:port",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
@@ -1580,7 +1579,6 @@ cc_test(
         ":packet",
         ":packet_test_cc_proto",
         ":type_map",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:core_proto",
         "//mediapipe/framework/port:gtest_main",
         "@com_google_absl//absl/strings",
@@ -1640,7 +1638,6 @@ cc_test(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework:status_handler_cc_proto",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:template_parser",

mediapipe/framework/api2/BUILD

@@ -36,7 +36,6 @@ cc_test(
         ":tag",
         ":test_contracts",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "@com_google_absl//absl/strings",
@@ -175,7 +174,6 @@ cc_test(
         ":port",
         ":test_contracts",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:subgraph_expansion",

mediapipe/framework/api2/builder.h

@@ -75,6 +75,9 @@ class TagIndexMap {
   std::map<std::string, std::vector<std::unique_ptr<T>>> map_;
 };
 
+class Graph;
+class NodeBase;
+
 // These structs are used internally to store information about the endpoints
 // of a connection.
 struct SourceBase;
@@ -109,7 +112,7 @@ class MultiPort : public Single {
 
 // These classes wrap references to the underlying source/destination
 // endpoints, adding type information and the user-visible API.
-template <bool AllowMultiple, bool IsSide, typename T = internal::Generic>
+template <bool IsSide, typename T = internal::Generic>
 class DestinationImpl {
  public:
   using Base = DestinationBase;
@@ -121,13 +124,12 @@ class DestinationImpl {
 };
 
 template <bool IsSide, typename T>
-class DestinationImpl<true, IsSide, T>
-    : public MultiPort<DestinationImpl<false, IsSide, T>> {
+class MultiDestinationImpl : public MultiPort<DestinationImpl<IsSide, T>> {
  public:
-  using MultiPort<DestinationImpl<false, IsSide, T>>::MultiPort;
+  using MultiPort<DestinationImpl<IsSide, T>>::MultiPort;
 };
 
-template <bool AllowMultiple, bool IsSide, typename T = internal::Generic>
+template <bool IsSide, typename T = internal::Generic>
 class SourceImpl {
  public:
   using Base = SourceBase;
@@ -135,9 +137,9 @@ class SourceImpl {
   // Src is used as the return type of fluent methods below. Since these are
   // single-port methods, it is desirable to always decay to a reference to the
   // single-port superclass, even if they are called on a multiport.
-  using Src = SourceImpl<false, IsSide, T>;
+  using Src = SourceImpl<IsSide, T>;
   template <typename U>
-  using Dst = DestinationImpl<false, IsSide, U>;
+  using Dst = DestinationImpl<IsSide, U>;
 
   // clang-format off
   template <typename U>
@@ -173,10 +175,9 @@ class SourceImpl {
 };
 
 template <bool IsSide, typename T>
-class SourceImpl<true, IsSide, T>
-    : public MultiPort<SourceImpl<false, IsSide, T>> {
+class MultiSourceImpl : public MultiPort<SourceImpl<IsSide, T>> {
  public:
-  using MultiPort<SourceImpl<false, IsSide, T>>::MultiPort;
+  using MultiPort<SourceImpl<IsSide, T>>::MultiPort;
 };
 
 // A source and a destination correspond to an output/input stream on a node,
@@ -185,14 +186,23 @@ class SourceImpl<true, IsSide, T>
 // For graph inputs/outputs, however, the inputs are sources, and the outputs
 // are destinations. This is because graph ports are connected "from inside"
 // when building the graph.
-template <bool AllowMultiple = false, typename T = internal::Generic>
-using Source = SourceImpl<AllowMultiple, false, T>;
-template <bool AllowMultiple = false, typename T = internal::Generic>
-using SideSource = SourceImpl<AllowMultiple, true, T>;
-template <bool AllowMultiple = false, typename T = internal::Generic>
-using Destination = DestinationImpl<AllowMultiple, false, T>;
-template <bool AllowMultiple = false, typename T = internal::Generic>
-using SideDestination = DestinationImpl<AllowMultiple, true, T>;
+template <typename T = internal::Generic>
+using Source = SourceImpl<false, T>;
+template <typename T = internal::Generic>
+using MultiSource = MultiSourceImpl<false, T>;
+template <typename T = internal::Generic>
+using SideSource = SourceImpl<true, T>;
+template <typename T = internal::Generic>
+using MultiSideSource = MultiSourceImpl<true, T>;
+
+template <typename T = internal::Generic>
+using Destination = DestinationImpl<false, T>;
+template <typename T = internal::Generic>
+using SideDestination = DestinationImpl<true, T>;
+template <typename T = internal::Generic>
+using MultiDestination = MultiDestinationImpl<false, T>;
+template <typename T = internal::Generic>
+using MultiSideDestination = MultiDestinationImpl<true, T>;
 
 class NodeBase {
  public:
@@ -202,45 +212,67 @@ class NodeBase {
   // of its entries by index. However, for nodes without visible contracts we
   // can't know whether a tag is indexable or not, so we would need the
   // multi-port to also be usable as a port directly (representing index 0).
-  Source<true> Out(const std::string& tag) {
-    return Source<true>(&out_streams_[tag]);
+  MultiSource<> Out(const std::string& tag) {
+    return MultiSource<>(&out_streams_[tag]);
   }
 
-  Destination<true> In(const std::string& tag) {
-    return Destination<true>(&in_streams_[tag]);
+  MultiDestination<> In(const std::string& tag) {
+    return MultiDestination<>(&in_streams_[tag]);
   }
 
-  SideSource<true> SideOut(const std::string& tag) {
-    return SideSource<true>(&out_sides_[tag]);
+  MultiSideSource<> SideOut(const std::string& tag) {
+    return MultiSideSource<>(&out_sides_[tag]);
   }
 
-  SideDestination<true> SideIn(const std::string& tag) {
-    return SideDestination<true>(&in_sides_[tag]);
+  MultiSideDestination<> SideIn(const std::string& tag) {
+    return MultiSideDestination<>(&in_sides_[tag]);
   }
 
   template <typename B, typename T, bool kIsOptional, bool kIsMultiple>
   auto operator[](const PortCommon<B, T, kIsOptional, kIsMultiple>& port) {
+    using PayloadT =
+        typename PortCommon<B, T, kIsOptional, kIsMultiple>::PayloadT;
     if constexpr (std::is_same_v<B, OutputBase>) {
-      return Source<kIsMultiple, T>(&out_streams_[port.Tag()]);
+      auto* base = &out_streams_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSource<PayloadT>(base);
+      } else {
+        return Source<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, InputBase>) {
-      return Destination<kIsMultiple, T>(&in_streams_[port.Tag()]);
+      auto* base = &in_streams_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiDestination<PayloadT>(base);
+      } else {
+        return Destination<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, SideOutputBase>) {
-      return SideSource<kIsMultiple, T>(&out_sides_[port.Tag()]);
+      auto* base = &out_sides_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSideSource<PayloadT>(base);
+      } else {
+        return SideSource<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, SideInputBase>) {
-      return SideDestination<kIsMultiple, T>(&in_sides_[port.Tag()]);
+      auto* base = &in_sides_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSideDestination<PayloadT>(base);
+      } else {
+        return SideDestination<PayloadT>(base);
+      }
     } else {
       static_assert(dependent_false<B>::value, "Type not supported.");
     }
   }
 
   // Convenience methods for accessing purely index-based ports.
-  Source<false> Out(int index) { return Out("")[index]; }
+  Source<> Out(int index) { return Out("")[index]; }
 
-  Destination<false> In(int index) { return In("")[index]; }
+  Destination<> In(int index) { return In("")[index]; }
 
-  SideSource<false> SideOut(int index) { return SideOut("")[index]; }
+  SideSource<> SideOut(int index) { return SideOut("")[index]; }
 
-  SideDestination<false> SideIn(int index) { return SideIn("")[index]; }
+  SideDestination<> SideIn(int index) { return SideIn("")[index]; }
 
   template <typename T>
   T& GetOptions() {
@@ -277,11 +309,6 @@ class Node<internal::Generic> : public NodeBase {
 
 using GenericNode = Node<internal::Generic>;
 
-template <template <bool, class> class BP, class Port, class TagIndexMapT>
-auto MakeBuilderPort(const Port& port, TagIndexMapT& streams) {
-  return BP<Port::kMultiple, typename Port::PayloadT>(&streams[port.Tag()]);
-}
-
 template <class Calc>
 class Node : public NodeBase {
  public:
@@ -298,25 +325,25 @@ class Node : public NodeBase {
   template <class Tag>
   auto Out(Tag tag) {
     constexpr auto& port = Calc::Contract::TaggedOutputs::get(tag);
-    return MakeBuilderPort<Source>(port, out_streams_);
+    return NodeBase::operator[](port);
   }
 
   template <class Tag>
   auto In(Tag tag) {
     constexpr auto& port = Calc::Contract::TaggedInputs::get(tag);
-    return MakeBuilderPort<Destination>(port, in_streams_);
+    return NodeBase::operator[](port);
   }
 
   template <class Tag>
   auto SideOut(Tag tag) {
     constexpr auto& port = Calc::Contract::TaggedSideOutputs::get(tag);
-    return MakeBuilderPort<SideSource>(port, out_sides_);
+    return NodeBase::operator[](port);
   }
 
   template <class Tag>
   auto SideIn(Tag tag) {
     constexpr auto& port = Calc::Contract::TaggedSideInputs::get(tag);
-    return MakeBuilderPort<SideDestination>(port, in_sides_);
+    return NodeBase::operator[](port);
   }
 
   // We could allow using the non-checked versions with typed nodes too, but
@@ -332,17 +359,17 @@ class PacketGenerator {
  public:
   PacketGenerator(std::string type) : type_(std::move(type)) {}
 
-  SideSource<true> SideOut(const std::string& tag) {
-    return SideSource<true>(&out_sides_[tag]);
+  MultiSideSource<> SideOut(const std::string& tag) {
+    return MultiSideSource<>(&out_sides_[tag]);
   }
 
-  SideDestination<true> SideIn(const std::string& tag) {
-    return SideDestination<true>(&in_sides_[tag]);
+  MultiSideDestination<> SideIn(const std::string& tag) {
+    return MultiSideDestination<>(&in_sides_[tag]);
   }
 
   // Convenience methods for accessing purely index-based ports.
-  SideSource<false> SideOut(int index) { return SideOut("")[index]; }
-  SideDestination<false> SideIn(int index) { return SideIn("")[index]; }
+  SideSource<> SideOut(int index) { return SideOut("")[index]; }
+  SideDestination<> SideIn(int index) { return SideIn("")[index]; }
 
   template <typename T>
   T& GetOptions() {
@@ -402,70 +429,85 @@ class Graph {
   }
 
   // Graph ports, non-typed.
-  Source<true> In(const std::string& graph_input) {
+  MultiSource<> In(const std::string& graph_input) {
     return graph_boundary_.Out(graph_input);
   }
 
-  Destination<true> Out(const std::string& graph_output) {
+  MultiDestination<> Out(const std::string& graph_output) {
     return graph_boundary_.In(graph_output);
   }
 
-  SideSource<true> SideIn(const std::string& graph_input) {
+  MultiSideSource<> SideIn(const std::string& graph_input) {
     return graph_boundary_.SideOut(graph_input);
   }
 
-  SideDestination<true> SideOut(const std::string& graph_output) {
+  MultiSideDestination<> SideOut(const std::string& graph_output) {
     return graph_boundary_.SideIn(graph_output);
   }
 
   // Convenience methods for accessing purely index-based ports.
-  Source<false> In(int index) { return In("")[0]; }
+  Source<> In(int index) { return In("")[index]; }
 
-  Destination<false> Out(int index) { return Out("")[0]; }
+  Destination<> Out(int index) { return Out("")[index]; }
 
-  SideSource<false> SideIn(int index) { return SideIn("")[0]; }
+  SideSource<> SideIn(int index) { return SideIn("")[index]; }
 
-  SideDestination<false> SideOut(int index) { return SideOut("")[0]; }
+  SideDestination<> SideOut(int index) { return SideOut("")[index]; }
 
   // Graph ports, typed.
   // TODO: make graph_boundary_ a typed node!
-  template <class PortT, class Payload = typename PortT::PayloadT,
-            class Src = Source<PortT::kMultiple, Payload>>
-  Src In(const PortT& graph_input) {
-    return Src(&graph_boundary_.out_streams_[graph_input.Tag()]);
+  template <class PortT, class Payload = typename PortT::PayloadT>
+  auto In(const PortT& graph_input) {
+    return (*this)[graph_input];
   }
 
-  template <class PortT, class Payload = typename PortT::PayloadT,
-            class Dst = Destination<PortT::kMultiple, Payload>>
-  Dst Out(const PortT& graph_output) {
-    return Dst(&graph_boundary_.in_streams_[graph_output.Tag()]);
+  template <class PortT, class Payload = typename PortT::PayloadT>
+  auto Out(const PortT& graph_output) {
+    return (*this)[graph_output];
   }
 
-  template <class PortT, class Payload = typename PortT::PayloadT,
-            class Src = SideSource<PortT::kMultiple, Payload>>
-  Src SideIn(const PortT& graph_input) {
-    return Src(&graph_boundary_.out_sides_[graph_input.Tag()]);
+  template <class PortT, class Payload = typename PortT::PayloadT>
+  auto SideIn(const PortT& graph_input) {
+    return (*this)[graph_input];
   }
 
-  template <class PortT, class Payload = typename PortT::PayloadT,
-            class Dst = SideDestination<PortT::kMultiple, Payload>>
-  Dst SideOut(const PortT& graph_output) {
-    return Dst(&graph_boundary_.in_sides_[graph_output.Tag()]);
+  template <class PortT, class Payload = typename PortT::PayloadT>
+  auto SideOut(const PortT& graph_output) {
+    return (*this)[graph_output];
   }
 
   template <typename B, typename T, bool kIsOptional, bool kIsMultiple>
   auto operator[](const PortCommon<B, T, kIsOptional, kIsMultiple>& port) {
+    using PayloadT =
+        typename PortCommon<B, T, kIsOptional, kIsMultiple>::PayloadT;
     if constexpr (std::is_same_v<B, OutputBase>) {
-      return Destination<kIsMultiple, T>(
-          &graph_boundary_.in_streams_[port.Tag()]);
+      auto* base = &graph_boundary_.in_streams_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiDestination<PayloadT>(base);
+      } else {
+        return Destination<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, InputBase>) {
-      return Source<kIsMultiple, T>(&graph_boundary_.out_streams_[port.Tag()]);
+      auto* base = &graph_boundary_.out_streams_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSource<PayloadT>(base);
+      } else {
+        return Source<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, SideOutputBase>) {
-      return SideDestination<kIsMultiple, T>(
-          &graph_boundary_.in_sides_[port.Tag()]);
+      auto* base = &graph_boundary_.in_sides_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSideDestination<PayloadT>(base);
+      } else {
+        return SideDestination<PayloadT>(base);
+      }
     } else if constexpr (std::is_same_v<B, SideInputBase>) {
-      return SideSource<kIsMultiple, T>(
-          &graph_boundary_.out_sides_[port.Tag()]);
+      auto* base = &graph_boundary_.out_sides_[port.Tag()];
+      if constexpr (kIsMultiple) {
+        return MultiSideSource<PayloadT>(base);
+      } else {
+        return SideSource<PayloadT>(base);
+      }
     } else {
       static_assert(dependent_false<B>::value, "Type not supported.");
     }

mediapipe/framework/api2/builder_test.cc

@@ -9,7 +9,6 @@
 #include "mediapipe/framework/api2/tag.h"
 #include "mediapipe/framework/api2/test_contracts.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
@@ -50,21 +49,21 @@ TEST(BuilderTest, BuildGraph) {
 
 TEST(BuilderTest, CopyableSource) {
   builder::Graph graph;
-  builder::Source<false, int> a = graph[Input<int>("A")];
+  builder::Source<int> a = graph[Input<int>("A")];
   a.SetName("a");
-  builder::Source<false, int> b = graph[Input<int>("B")];
+  builder::Source<int> b = graph[Input<int>("B")];
   b.SetName("b");
-  builder::SideSource<false, float> side_a = graph[SideInput<float>("SIDE_A")];
+  builder::SideSource<float> side_a = graph[SideInput<float>("SIDE_A")];
   side_a.SetName("side_a");
-  builder::SideSource<false, float> side_b = graph[SideInput<float>("SIDE_B")];
+  builder::SideSource<float> side_b = graph[SideInput<float>("SIDE_B")];
   side_b.SetName("side_b");
-  builder::Destination<false, int> out = graph[Output<int>("OUT")];
-  builder::SideDestination<false, float> side_out =
+  builder::Destination<int> out = graph[Output<int>("OUT")];
+  builder::SideDestination<float> side_out =
       graph[SideOutput<float>("SIDE_OUT")];
 
-  builder::Source<false, int> input = a;
+  builder::Source<int> input = a;
   input = b;
-  builder::SideSource<false, float> side_input = side_b;
+  builder::SideSource<float> side_input = side_b;
   side_input = side_a;
 
   input >> out;
@@ -85,27 +84,26 @@ TEST(BuilderTest, CopyableSource) {
 TEST(BuilderTest, BuildGraphWithFunctions) {
   builder::Graph graph;
 
-  builder::Source<false, int> base = graph[Input<int>("IN")];
+  builder::Source<int> base = graph[Input<int>("IN")];
   base.SetName("base");
-  builder::SideSource<false, float> side = graph[SideInput<float>("SIDE")];
+  builder::SideSource<float> side = graph[SideInput<float>("SIDE")];
   side.SetName("side");
 
-  auto foo_fn = [](builder::Source<false, int> base,
-                   builder::SideSource<false, float> side,
+  auto foo_fn = [](builder::Source<int> base, builder::SideSource<float> side,
                    builder::Graph& graph) {
     auto& foo = graph.AddNode("Foo");
     base >> foo[Input<int>("BASE")];
     side >> foo[SideInput<float>("SIDE")];
     return foo[Output<double>("OUT")];
   };
-  builder::Source<false, double> foo_out = foo_fn(base, side, graph);
+  builder::Source<double> foo_out = foo_fn(base, side, graph);
 
-  auto bar_fn = [](builder::Source<false, double> in, builder::Graph& graph) {
+  auto bar_fn = [](builder::Source<double> in, builder::Graph& graph) {
     auto& bar = graph.AddNode("Bar");
     in >> bar[Input<double>("IN")];
     return bar[Output<double>("OUT")];
   };
-  builder::Source<false, double> bar_out = bar_fn(foo_out, graph);
+  builder::Source<double> bar_out = bar_fn(foo_out, graph);
   bar_out.SetName("out");
 
   bar_out >> graph[Output<double>("OUT")];
@@ -298,6 +296,34 @@ TEST(BuilderTest, EmptyTag) {
   EXPECT_THAT(graph.GetConfig(), EqualsProto(expected));
 }
 
+TEST(BuilderTest, GraphIndexes) {
+  builder::Graph graph;
+  auto& foo = graph.AddNode("Foo");
+  graph.In(0).SetName("a") >> foo.In("")[0];
+  graph.In(1).SetName("c") >> foo.In("")[2];
+  graph.In(2).SetName("b") >> foo.In("")[1];
+  foo.Out("")[0].SetName("x") >> graph.Out(1);
+  foo.Out("")[1].SetName("y") >> graph.Out(0);
+
+  CalculatorGraphConfig expected =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: "a"
+        input_stream: "c"
+        input_stream: "b"
+        output_stream: "y"
+        output_stream: "x"
+        node {
+          calculator: "Foo"
+          input_stream: "a"
+          input_stream: "b"
+          input_stream: "c"
+          output_stream: "x"
+          output_stream: "y"
+        }
+      )pb");
+  EXPECT_THAT(graph.GetConfig(), EqualsProto(expected));
+}
+
 }  // namespace test
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/api2/node_test.cc

@@ -565,6 +565,19 @@ TEST(NodeTest, ConsumeInputs) {
   MP_EXPECT_OK(graph.WaitUntilDone());
 }
 
+// Just to test that single-port contracts work.
+struct LogSinkNode : public Node {
+  static constexpr Input<int> kIn{"IN"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn);
+
+  absl::Status Process(CalculatorContext* cc) override {
+    LOG(INFO) << "LogSinkNode received: " << kIn(cc).Get();
+    return {};
+  }
+};
+MEDIAPIPE_REGISTER_NODE(LogSinkNode);
+
 }  // namespace test
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/api2/subgraph_test.cc

@@ -4,7 +4,6 @@
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/api2/test_contracts.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"

mediapipe/framework/api2/tuple.h

@@ -18,11 +18,10 @@ template <class Tuple>
 using tuple_index_sequence =
     std::make_index_sequence<std::tuple_size_v<std::decay_t<Tuple>>>;
 
-// Concatenates two std::index_sequences.
-template <std::size_t... I, std::size_t... J>
-constexpr auto index_sequence_cat(std::index_sequence<I...>,
-                                  std::index_sequence<J...>)
-    -> std::index_sequence<I..., J...> {
+// Concatenates multiple std::index_sequences.
+template <std::size_t... I>
+constexpr auto index_sequence_cat(std::index_sequence<I...>)
+    -> std::index_sequence<I...> {
   return {};
 }
 

mediapipe/framework/calculator_base_test.cc

@@ -174,7 +174,7 @@ TEST(CalculatorTest, CreateByName) {
 TEST(CalculatorTest, CreateByNameWhitelisted) {
   // Reset the registration namespace whitelist.
   *const_cast<absl::flat_hash_set<std::string>*>(
-      &NamespaceWhitelist::TopNamespaces()) = absl::flat_hash_set<std::string>{
+      &NamespaceAllowlist::TopNamespaces()) = absl::flat_hash_set<std::string>{
       "mediapipe::test_ns::whitelisted_ns",
       "mediapipe",
   };

mediapipe/framework/calculator_graph.cc

@@ -427,7 +427,8 @@ absl::Status CalculatorGraph::Initialize(
     const std::map<std::string, Packet>& side_packets) {
   auto validated_graph = absl::make_unique<ValidatedGraphConfig>();
   MP_RETURN_IF_ERROR(validated_graph->Initialize(
-      input_config, /*graph_registry=*/nullptr, &service_manager_));
+      input_config, /*graph_registry=*/nullptr, /*graph_options=*/nullptr,
+      &service_manager_));
   return Initialize(std::move(validated_graph), side_packets);
 }
 

mediapipe/framework/calculator_node.h

@@ -236,21 +236,21 @@ class CalculatorNode {
   }
 
  private:
-  // Sets up the output side packets from the master flat array.
+  // Sets up the output side packets from the main flat array.
   absl::Status InitializeOutputSidePackets(
       const PacketTypeSet& output_side_packet_types,
       OutputSidePacketImpl* output_side_packets);
   // Connects the input side packets as mirrors on the output side packets.
-  // Output side packets are looked up in the master flat array which is
+  // Output side packets are looked up in the main flat array which is
   // provided.
   absl::Status InitializeInputSidePackets(
       OutputSidePacketImpl* output_side_packets);
-  // Sets up the output streams from the master flat array.
+  // Sets up the output streams from the main flat array.
   absl::Status InitializeOutputStreams(
       OutputStreamManager* output_stream_managers);
   // Sets up the input streams and connects them as mirrors on the
   // output streams.  Both input streams and output streams are looked
-  // up in the master flat arrays which are provided.
+  // up in the main flat arrays which are provided.
   absl::Status InitializeInputStreams(
       InputStreamManager* input_stream_managers,
       OutputStreamManager* output_stream_managers);

mediapipe/framework/calculator_runner.h

@@ -122,7 +122,7 @@ class CalculatorRunner {
   const StreamContentsSet& Outputs() const { return *outputs_; }
 
   // Returns the access to the output side packets.
-  const PacketSet& OutputSidePackets() { return *output_side_packets_.get(); }
+  const PacketSet& OutputSidePackets() { return *output_side_packets_; }
 
   // Returns a graph counter.
   mediapipe::Counter* GetCounter(const std::string& name);

mediapipe/framework/collection.h

@@ -26,6 +26,7 @@
 #include "absl/base/macros.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/tool/tag_map.h"
@@ -50,7 +51,7 @@ struct CollectionErrorHandlerFatal {
   // Since there isn't any state and we're not returning anything, we
   // get away with only one version of this function (which is const
   // but returns a non-const reference).
-  T& GetFallback(const std::string& tag, int index) const {
+  T& GetFallback(const absl::string_view tag, int index) const {
     LOG(FATAL) << "Failed to get tag \"" << tag << "\" index " << index;
     std::abort();
   }
@@ -131,16 +132,16 @@ class Collection {
   const value_type& Get(CollectionItemId id) const;
 
   // Convenience functions.
-  value_type& Get(const std::string& tag, int index);
-  const value_type& Get(const std::string& tag, int index) const;
+  value_type& Get(absl::string_view tag, int index);
+  const value_type& Get(absl::string_view tag, int index) const;
 
   // Equivalent to Get("", index);
   value_type& Index(int index);
   const value_type& Index(int index) const;
 
   // Equivalent to Get(tag, 0);
-  value_type& Tag(const std::string& tag);
-  const value_type& Tag(const std::string& tag) const;
+  value_type& Tag(absl::string_view tag);
+  const value_type& Tag(absl::string_view tag) const;
 
   // These functions only exist for collections with storage ==
   // kStorePointer.  GetPtr returns the stored ptr value rather than
@@ -179,13 +180,15 @@ class Collection {
   ////////////////////////////////////////
 
   // Returns true if the provided tag is available (not necessarily set yet).
-  bool HasTag(const std::string& tag) const { return tag_map_->HasTag(tag); }
+  bool HasTag(const absl::string_view tag) const {
+    return tag_map_->HasTag(tag);
+  }
 
   // Returns the number of entries in this collection.
   int NumEntries() const { return tag_map_->NumEntries(); }
 
   // Returns the number of entries with the provided tag.
-  int NumEntries(const std::string& tag) const {
+  int NumEntries(const absl::string_view tag) const {
     return tag_map_->NumEntries(tag);
   }
 
@@ -200,7 +203,7 @@ class Collection {
   // However, be careful in using this fact, as it circumvents the
   // validity checks in GetId() (i.e. ++GetId("BLAH", 2) looks like it
   // is valid, while GetId("BLAH", 3) is not valid).
-  CollectionItemId GetId(const std::string& tag, int index) const {
+  CollectionItemId GetId(const absl::string_view tag, int index) const {
     return tag_map_->GetId(tag, index);
   }
 
@@ -234,10 +237,10 @@ class Collection {
   //   for (CollectionItemId id = collection.BeginId(tag);
   //        id < collection.EndId(tag); ++id) {
   //   }
-  CollectionItemId BeginId(const std::string& tag) const {
+  CollectionItemId BeginId(const absl::string_view tag) const {
     return tag_map_->BeginId(tag);
   }
-  CollectionItemId EndId(const std::string& tag) const {
+  CollectionItemId EndId(const absl::string_view tag) const {
     return tag_map_->EndId(tag);
   }
 
@@ -404,7 +407,7 @@ bool Collection<T, storage, ErrorHandler>::UsesTags() const {
     return false;
   }
   // If the one tag present is non-empty then we are using tags.
-  return mapping.begin()->first != "";
+  return !mapping.begin()->first.empty();
 }
 
 template <typename T, CollectionStorage storage, typename ErrorHandler>
@@ -449,7 +452,8 @@ Collection<T, storage, ErrorHandler>::GetPtr(CollectionItemId id) const {
 
 template <typename T, CollectionStorage storage, typename ErrorHandler>
 typename Collection<T, storage, ErrorHandler>::value_type&
-Collection<T, storage, ErrorHandler>::Get(const std::string& tag, int index) {
+Collection<T, storage, ErrorHandler>::Get(const absl::string_view tag,
+                                          int index) {
   CollectionItemId id = GetId(tag, index);
   if (!id.IsValid()) {
     return error_handler_.GetFallback(tag, index);
@@ -459,7 +463,7 @@ Collection<T, storage, ErrorHandler>::Get(const std::string& tag, int index) {
 
 template <typename T, CollectionStorage storage, typename ErrorHandler>
 const typename Collection<T, storage, ErrorHandler>::value_type&
-Collection<T, storage, ErrorHandler>::Get(const std::string& tag,
+Collection<T, storage, ErrorHandler>::Get(const absl::string_view tag,
                                           int index) const {
   CollectionItemId id = GetId(tag, index);
   if (!id.IsValid()) {
@@ -482,13 +486,13 @@ Collection<T, storage, ErrorHandler>::Index(int index) const {
 
 template <typename T, CollectionStorage storage, typename ErrorHandler>
 typename Collection<T, storage, ErrorHandler>::value_type&
-Collection<T, storage, ErrorHandler>::Tag(const std::string& tag) {
+Collection<T, storage, ErrorHandler>::Tag(const absl::string_view tag) {
   return Get(tag, 0);
 }
 
 template <typename T, CollectionStorage storage, typename ErrorHandler>
 const typename Collection<T, storage, ErrorHandler>::value_type&
-Collection<T, storage, ErrorHandler>::Tag(const std::string& tag) const {
+Collection<T, storage, ErrorHandler>::Tag(const absl::string_view tag) const {
   return Get(tag, 0);
 }
 
@@ -535,21 +539,23 @@ Collection<T, storage, ErrorHandler>::end() const {
 // Returns c.HasTag(tag) && !Tag(tag)->IsEmpty() (just for convenience).
 // This version is used with Calculator.
 template <class S>
-bool HasTagValue(const internal::Collection<S*>& c, const std::string& tag) {
+bool HasTagValue(const internal::Collection<S*>& c,
+                 const absl::string_view tag) {
   return c.HasTag(tag) && !c.Tag(tag)->IsEmpty();
 }
 
 // Returns c.HasTag(tag) && !Tag(tag).IsEmpty() (just for convenience).
 // This version is used with CalculatorBase.
 template <class S>
-bool HasTagValue(const internal::Collection<S>& c, const std::string& tag) {
+bool HasTagValue(const internal::Collection<S>& c,
+                 const absl::string_view tag) {
   return c.HasTag(tag) && !c.Tag(tag).IsEmpty();
 }
 
 // Returns c.HasTag(tag) && !Tag(tag).IsEmpty() (just for convenience).
 // This version is used with Calculator or CalculatorBase.
 template <class C>
-bool HasTagValue(const C& c, const std::string& tag) {
+bool HasTagValue(const C& c, const absl::string_view tag) {
   return HasTagValue(c->Inputs(), tag);
 }
 

mediapipe/framework/deps/BUILD

@@ -87,10 +87,11 @@ cc_library(
     name = "message_matchers",
     testonly = True,
     hdrs = ["message_matchers.h"],
-    visibility = ["//visibility:public"],
+    # Use this library through "mediapipe/framework/port:gtest_main".
+    visibility = ["//mediapipe/framework/port:__pkg__"],
     deps = [
         "//mediapipe/framework/port:core_proto",
-        "//mediapipe/framework/port:gtest_main",
+        "@com_google_googletest//:gtest",
     ],
 )
 

mediapipe/framework/deps/message_matchers.h

@@ -17,8 +17,8 @@
 
 #include <memory>
 
+#include "gmock/gmock.h"
 #include "mediapipe/framework/port/core_proto_inc.h"
-#include "mediapipe/framework/port/gmock.h"
 
 namespace mediapipe {
 

mediapipe/framework/deps/registration.cc

@@ -21,7 +21,7 @@ namespace mediapipe {
 namespace {
 
 // List of namespaces that can register calculators inside the namespace
-// and still refer to them using an unqualified name.  This whitelist
+// and still refer to them using an unqualified name.  This allowlist
 // is meant to facilitate migration from unqualified to fully qualified
 // calculator names.
 constexpr char const* kTopNamespaces[] = {
@@ -36,7 +36,7 @@ inline size_t array_size(T (&arr)[SIZE]) {
 }  // namespace
 
 /*static*/
-const absl::flat_hash_set<std::string>& NamespaceWhitelist::TopNamespaces() {
+const absl::flat_hash_set<std::string>& NamespaceAllowlist::TopNamespaces() {
   static absl::flat_hash_set<std::string>* result =
       new absl::flat_hash_set<std::string>(
           kTopNamespaces, kTopNamespaces + array_size(kTopNamespaces));

mediapipe/framework/deps/registration.h

@@ -144,7 +144,7 @@ struct WrapStatusOr<absl::StatusOr<T>> {
 };
 }  // namespace registration_internal
 
-class NamespaceWhitelist {
+class NamespaceAllowlist {
  public:
   static const absl::flat_hash_set<std::string>& TopNamespaces();
 };
@@ -289,14 +289,14 @@ class FunctionRegistry {
   mutable absl::Mutex lock_;
   std::unordered_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
 
-  // For names included in NamespaceWhitelist, strips the namespace.
+  // For names included in NamespaceAllowlist, strips the namespace.
   std::string GetAdjustedName(const std::string& name) {
     constexpr auto kCxxSep = registration_internal::kCxxSep;
     std::vector<std::string> names = absl::StrSplit(name, kCxxSep);
     std::string base_name = names.back();
     names.pop_back();
     std::string ns = absl::StrJoin(names, kCxxSep);
-    if (NamespaceWhitelist::TopNamespaces().count(ns)) {
+    if (NamespaceAllowlist::TopNamespaces().count(ns)) {
       return base_name;
     }
     return name;

mediapipe/framework/formats/body_rig.proto

@@ -0,0 +1,40 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+// Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of
+// the joint and its visibility.
+message Joint {
+  // Joint rotation in 6D contineous representation.
+  // Such representation is more sutable for NN model training and can be
+  // converted to quaternions and Euler angles if needed. Details can be found
+  // in https://arxiv.org/abs/1812.07035.
+  repeated float rotation_6d = 1;
+
+  // Joint visibility.
+  // Float score of whether joint is visible: present on the screen and not
+  // occluded by other objects. Depending on the model, visibility value is
+  // either a sigmoid or an argument of sigmoid, but in any case higher value
+  // indicates higher probability of joint being visible. Should stay unset if
+  // not supported.
+  optional float visibility = 2;
+}
+
+// Group of Joint protos.
+message JointList {
+  repeated Joint joint = 1;
+}

mediapipe/framework/formats/image.cc

@@ -77,13 +77,6 @@ bool Image::ConvertToGpu() const {
 #else
   // GlCalculatorHelperImpl::MakeGlTextureBuffer (CreateSourceTexture)
   auto buffer = mediapipe::GlTextureBuffer::Create(*image_frame_);
-  glBindTexture(GL_TEXTURE_2D, buffer->name());
-  // See GlCalculatorHelperImpl::SetStandardTextureParams
-  glTexParameteri(buffer->target(), GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-  glTexParameteri(buffer->target(), GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-  glTexParameteri(buffer->target(), GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-  glTexParameteri(buffer->target(), GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-  glBindTexture(GL_TEXTURE_2D, 0);
   glFlush();
   gpu_buffer_ = mediapipe::GpuBuffer(std::move(buffer));
 #endif  //  MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER

mediapipe/framework/formats/image.h

@@ -109,8 +109,7 @@ class Image {
     return gpu_buffer_.GetCVPixelBufferRef();
   }
 #else
-  const mediapipe::GlTextureBufferSharedPtr& GetGlTextureBufferSharedPtr()
-      const {
+  mediapipe::GlTextureBufferSharedPtr GetGlTextureBufferSharedPtr() const {
     if (use_gpu_ == false) ConvertToGpu();
     return gpu_buffer_.GetGlTextureBufferSharedPtr();
   }

mediapipe/framework/formats/image_frame.cc

@@ -242,7 +242,7 @@ bool ImageFrame::IsValidAlignmentNumber(uint32 alignment_boundary) {
 
 // static
 std::string ImageFrame::InvalidFormatString(ImageFormat::Format format) {
-#ifdef MEDIAPIPE_MOBILE
+#ifdef MEDIAPIPE_PROTO_LITE
   return "Invalid format.";
 #else
   const proto_ns::EnumValueDescriptor* enum_value_descriptor =

mediapipe/framework/formats/image_frame_opencv.cc

@@ -75,7 +75,6 @@ int GetMatType(const mediapipe::ImageFormat::Format format) {
 }  // namespace
 
 namespace mediapipe {
-
 namespace formats {
 
 cv::Mat MatView(const ImageFrame* image) {

mediapipe/framework/formats/image_opencv.cc

@@ -75,7 +75,6 @@ int GetMatType(const mediapipe::ImageFormat::Format format) {
 }
 }  // namespace
 namespace mediapipe {
-
 namespace formats {
 
 cv::Mat MatView(const mediapipe::Image* image) {

mediapipe/framework/formats/tensor_internal.h

@@ -0,0 +1,42 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_TENSOR_INTERNAL_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_TENSOR_INTERNAL_H_
+
+#include <cstdint>
+
+namespace mediapipe {
+
+// Generates unique view id at compile-time using FILE and LINE.
+#define TENSOR_UNIQUE_VIEW_TYPE_ID()                          \
+  static constexpr uint64_t kId = tensor_internal::FnvHash64( \
+      __FILE__, tensor_internal::FnvHash64(TENSOR_INT_TO_STRING(__LINE__)))
+
+namespace tensor_internal {
+
+#define TENSOR_INT_TO_STRING2(x) #x
+#define TENSOR_INT_TO_STRING(x) TENSOR_INT_TO_STRING2(x)
+
+// Compile-time hash function
+// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+constexpr uint64_t kFnvPrime = 0x00000100000001B3;
+constexpr uint64_t kFnvOffsetBias = 0xcbf29ce484222325;
+constexpr uint64_t FnvHash64(const char* str, uint64_t hash = kFnvOffsetBias) {
+  return (str[0] == 0) ? hash : FnvHash64(str + 1, (hash ^ str[0]) * kFnvPrime);
+}
+}  // namespace tensor_internal
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_TENSOR_INTERNAL_H_

mediapipe/framework/graph_validation_test.cc

@@ -20,7 +20,6 @@
 #include <functional>
 
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"

mediapipe/framework/input_stream_handler.cc

@@ -21,7 +21,6 @@
 #include "mediapipe/framework/port/ret_check.h"
 
 namespace mediapipe {
-
 using SyncSet = InputStreamHandler::SyncSet;
 
 absl::Status InputStreamHandler::InitializeInputStreamManagers(

mediapipe/framework/input_stream_handler.h

@@ -82,7 +82,7 @@ class InputStreamHandler {
   // flat_input_stream_managers is expected to point to a contiguous
   // flat array with InputStreamManagers corresponding to the id's in
   // InputStreamHandler::input_stream_managers_ (meaning it should point
-  // to somewhere in the middle of the master flat array of all input
+  // to somewhere in the middle of the main flat array of all input
   // stream managers).
   absl::Status InitializeInputStreamManagers(
       InputStreamManager* flat_input_stream_managers);

mediapipe/framework/output_stream_handler.h

@@ -74,7 +74,7 @@ class OutputStreamHandler {
   // flat_output_stream_managers is expected to point to a contiguous
   // flat array with OutputStreamManagers corresponding to the id's in
   // OutputStreamHandler::output_stream_managers_ (meaning it should
-  // point to somewhere in the middle of the master flat array of all
+  // point to somewhere in the middle of the main flat array of all
   // output stream managers).
   absl::Status InitializeOutputStreamManagers(
       OutputStreamManager* flat_output_stream_managers);

mediapipe/framework/packet.cc

@@ -21,6 +21,7 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/framework/tool/type_util.h"
 
 namespace mediapipe {
 namespace packet_internal {
@@ -105,6 +106,22 @@ std::string Packet::DebugString() const {
   return result;
 }
 
+absl::Status Packet::ValidateAsType(const tool::TypeInfo& type_info) const {
+  if (ABSL_PREDICT_FALSE(IsEmpty())) {
+    return absl::InternalError(
+        absl::StrCat("Expected a Packet of type: ",
+                     MediaPipeTypeStringOrDemangled(type_info),
+                     ", but received an empty Packet."));
+  }
+  bool holder_is_right_type = holder_->GetTypeId() == type_info.hash_code();
+  if (ABSL_PREDICT_FALSE(!holder_is_right_type)) {
+    return absl::InvalidArgumentError(absl::StrCat(
+        "The Packet stores \"", holder_->DebugTypeName(), "\", but \"",
+        MediaPipeTypeStringOrDemangled(type_info), "\" was requested."));
+  }
+  return absl::OkStatus();
+}
+
 absl::Status Packet::ValidateAsProtoMessageLite() const {
   if (ABSL_PREDICT_FALSE(IsEmpty())) {
     return absl::InternalError("Packet is empty.");

mediapipe/framework/packet.h

@@ -179,7 +179,9 @@ class Packet {
 
   // Returns an error if the packet does not contain data of type T.
   template <typename T>
-  absl::Status ValidateAsType() const;
+  absl::Status ValidateAsType() const {
+    return ValidateAsType(tool::TypeId<T>());
+  }
 
   // Returns an error if the packet is not an instance of
   // a protocol buffer message.
@@ -218,6 +220,8 @@ class Packet {
   friend std::shared_ptr<packet_internal::HolderBase>
   packet_internal::GetHolderShared(Packet&& packet);
 
+  absl::Status ValidateAsType(const tool::TypeInfo& type_info) const;
+
   std::shared_ptr<packet_internal::HolderBase> holder_;
   class Timestamp timestamp_;
 };
@@ -359,13 +363,8 @@ class HolderBase {
   HolderBase& operator=(const HolderBase&) = delete;
   virtual ~HolderBase();
   template <typename T>
-  void SetHolderTypeId() {
-    type_id_ = tool::GetTypeHash<T>();
-  }
-  size_t GetHolderTypeId() const { return type_id_; }
-  template <typename T>
-  bool HolderIsOfType() const {
-    return type_id_ == tool::GetTypeHash<T>();
+  bool PayloadIsOfType() const {
+    return GetTypeId() == tool::GetTypeHash<T>();
   }
   // Returns a printable std::string identifying the type stored in the holder.
   virtual const std::string DebugTypeName() const = 0;
@@ -393,8 +392,7 @@ class HolderBase {
   virtual StatusOr<std::vector<const proto_ns::MessageLite*>>
   GetVectorOfProtoMessageLite() const = 0;
 
- private:
-  size_t type_id_;
+  virtual bool HasForeignOwner() const { return false; }
 };
 
 // Two helper functions to get the proto base pointers.
@@ -501,7 +499,6 @@ class Holder : public HolderBase {
  public:
   explicit Holder(const T* ptr) : ptr_(ptr) {
     HolderSupport<T>::EnsureStaticInit();
-    SetHolderTypeId<Holder>();
   }
   ~Holder() override { delete_helper(); }
   const T& data() const {
@@ -517,9 +514,7 @@ class Holder : public HolderBase {
   absl::StatusOr<std::unique_ptr<T>> Release(
       typename std::enable_if<!std::is_array<U>::value ||
                               std::extent<U>::value != 0>::type* = 0) {
-    // Since C++ doesn't allow virtual, templated functions, check holder
-    // type here to make sure it's not upcasted from a ForeignHolder.
-    if (!HolderIsOfType<Holder<T>>()) {
+    if (HasForeignOwner()) {
       return InternalError(
           "Foreign holder can't release data ptr without ownership.");
     }
@@ -588,25 +583,19 @@ class Holder : public HolderBase {
 template <typename T>
 class ForeignHolder : public Holder<T> {
  public:
-  explicit ForeignHolder(const T* ptr) : Holder<T>(ptr) {
-    // Distinguishes between Holder and ForeignHolder since Consume() treats
-    // them differently.
-    this->template SetHolderTypeId<ForeignHolder>();
-  }
+  using Holder<T>::Holder;
   ~ForeignHolder() override {
     // Null out ptr_ so it doesn't get deleted by ~Holder.
+    // Note that ~Holder cannot call HasForeignOwner because the subclass's
+    // destructor runs first.
     this->ptr_ = nullptr;
   }
-  // Foreign holder can't release data pointer without ownership.
-  absl::StatusOr<std::unique_ptr<T>> Release() {
-    return absl::InternalError(
-        "Foreign holder can't release data ptr without ownership.");
-  }
+  bool HasForeignOwner() const final { return true; }
 };
 
 template <typename T>
 Holder<T>* HolderBase::As() {
-  if (HolderIsOfType<Holder<T>>() || HolderIsOfType<ForeignHolder<T>>()) {
+  if (PayloadIsOfType<T>()) {
     return static_cast<Holder<T>*>(this);
   }
   // Does not hold a T.
@@ -615,7 +604,7 @@ Holder<T>* HolderBase::As() {
 
 template <typename T>
 const Holder<T>* HolderBase::As() const {
-  if (HolderIsOfType<Holder<T>>() || HolderIsOfType<ForeignHolder<T>>()) {
+  if (PayloadIsOfType<T>()) {
     return static_cast<const Holder<T>*>(this);
   }
   // Does not hold a T.
@@ -644,7 +633,7 @@ inline absl::StatusOr<std::unique_ptr<T>> Packet::Consume() {
   MP_RETURN_IF_ERROR(ValidateAsType<T>());
   // Clients who use this function are responsible for ensuring that no
   // other thread is doing anything with this Packet.
-  if (holder_.unique()) {
+  if (!holder_->HasForeignOwner() && holder_.unique()) {
     VLOG(2) << "Consuming the data of " << DebugString();
     absl::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
@@ -666,8 +655,7 @@ inline absl::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
     typename std::enable_if<!std::is_array<T>::value>::type*) {
   MP_RETURN_IF_ERROR(ValidateAsType<T>());
   // If holder is the sole owner of the underlying data, consumes this packet.
-  if (!holder_->HolderIsOfType<packet_internal::ForeignHolder<T>>() &&
-      holder_.unique()) {
+  if (!holder_->HasForeignOwner() && holder_.unique()) {
     VLOG(2) << "Consuming the data of " << DebugString();
     absl::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
@@ -697,8 +685,7 @@ inline absl::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
                             std::extent<T>::value != 0>::type*) {
   MP_RETURN_IF_ERROR(ValidateAsType<T>());
   // If holder is the sole owner of the underlying data, consumes this packet.
-  if (!holder_->HolderIsOfType<packet_internal::ForeignHolder<T>>() &&
-      holder_.unique()) {
+  if (!holder_->HasForeignOwner() && holder_.unique()) {
     VLOG(2) << "Consuming the data of " << DebugString();
     absl::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
@@ -770,21 +757,6 @@ inline const T& Packet::Get() const {
   return holder->data();
 }
 
-template <typename T>
-absl::Status Packet::ValidateAsType() const {
-  if (ABSL_PREDICT_FALSE(IsEmpty())) {
-    return absl::InternalError(absl::StrCat(
-        "Expected a Packet of type: ", MediaPipeTypeStringOrDemangled<T>(),
-        ", but received an empty Packet."));
-  }
-  if (ABSL_PREDICT_FALSE(holder_->As<T>() == nullptr)) {
-    return absl::InvalidArgumentError(absl::StrCat(
-        "The Packet stores \"", holder_->DebugTypeName(), "\", but \"",
-        MediaPipeTypeStringOrDemangled<T>(), "\" was requested."));
-  }
-  return absl::OkStatus();
-}
-
 inline Timestamp Packet::Timestamp() const { return timestamp_; }
 
 template <typename T>

mediapipe/framework/packet_test.cc

@@ -21,7 +21,6 @@
 #include <vector>
 
 #include "absl/strings/str_cat.h"
-#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/packet_test.pb.h"
 #include "mediapipe/framework/port/core_proto_inc.h"
 #include "mediapipe/framework/port/gmock.h"
@@ -374,9 +373,9 @@ TEST(PacketTest, TestConsumeForeignHolder) {
   Packet packet = PointToForeign(data.get());
   absl::StatusOr<std::unique_ptr<int>> result = packet.Consume<int>();
   EXPECT_FALSE(result.ok());
-  EXPECT_EQ(result.status().code(), absl::StatusCode::kInternal);
+  EXPECT_EQ(result.status().code(), absl::StatusCode::kFailedPrecondition);
   EXPECT_EQ(result.status().message(),
-            "Foreign holder can't release data ptr without ownership.");
+            "Packet isn't the sole owner of the holder.");
   ASSERT_FALSE(packet.IsEmpty());
   EXPECT_EQ(33, packet.Get<int>());
 }