Project import generated by Copybara.

GitOrigin-RevId: 373e3ac1e5839befd95bf7d73ceff3c5f1171969

WORKSPACE

@@ -16,11 +16,11 @@ bazel_skylib_workspace()
 load("@bazel_skylib//lib:versions.bzl", "versions")
 versions.check(minimum_bazel_version = "3.7.2")
 
-# ABSL cpp library lts_2020_09_23
+# ABSL cpp library lts_2021_03_24, patch 2.
 http_archive(
     name = "com_google_absl",
     urls = [
-        "https://github.com/abseil/abseil-cpp/archive/20200923.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz",
     ],
     # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
     patches = [
@@ -29,8 +29,8 @@ http_archive(
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "abseil-cpp-20200923",
-    sha256 = "b3744a4f7a249d5eaf2309daad597631ce77ea62e0fc6abffbab4b4c3dc0fc08"
+    strip_prefix = "abseil-cpp-20210324.2",
+    sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f"
 )
 
 http_archive(
@@ -157,11 +157,11 @@ http_archive(
 http_archive(
     name = "pybind11",
     urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
-        "https://github.com/pybind/pybind11/archive/v2.4.3.tar.gz",
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
+        "https://github.com/pybind/pybind11/archive/v2.7.1.tar.gz",
     ],
-    sha256 = "1eed57bc6863190e35637290f97a20c81cfe4d9090ac0a24f3bbf08f265eb71d",
-    strip_prefix = "pybind11-2.4.3",
+    sha256 = "616d1c42e4cf14fa27b2a4ff759d7d7b33006fdc5ad8fd603bb2c22622f27020",
+    strip_prefix = "pybind11-2.7.1",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
@@ -333,6 +333,7 @@ maven_install(
         "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
         "androidx.lifecycle:lifecycle-common:2.3.1",
         "androidx.activity:activity:1.2.2",
+        "androidx.exifinterface:exifinterface:1.3.3",
         "androidx.fragment:fragment:1.3.4",
         "androidx.annotation:annotation:aar:1.1.0",
         "androidx.appcompat:appcompat:aar:1.1.0-rc01",
@@ -349,8 +350,8 @@ maven_install(
         "com.google.auto.value:auto-value:1.8.1",
         "com.google.auto.value:auto-value-annotations:1.8.1",
         "com.google.code.findbugs:jsr305:latest.release",
-        "com.google.flogger:flogger-system-backend:latest.release",
-        "com.google.flogger:flogger:latest.release",
+        "com.google.flogger:flogger-system-backend:0.6",
+        "com.google.flogger:flogger:0.6",
         "com.google.guava:guava:27.0.1-android",
         "com.google.guava:listenablefuture:1.0",
         "junit:junit:4.12",
@@ -389,6 +390,8 @@ http_archive(
     patches = [
         "@//third_party:org_tensorflow_compatibility_fixes.diff",
         "@//third_party:org_tensorflow_objc_cxx17.diff",
+        # Diff is generated with a script, don't update it manually.
+        "@//third_party:org_tensorflow_custom_ops.diff",
     ],
     patch_args = [
         "-p1",

docs/getting_started/android_solutions.md

@@ -26,15 +26,17 @@ the following into the project's Gradle dependencies:
 
 ```
 dependencies {
-    // MediaPipe solution-core is the foundation of any MediaPipe solutions.
+    // MediaPipe solution-core is the foundation of any MediaPipe Solutions.
     implementation 'com.google.mediapipe:solution-core:latest.release'
-    // Optional: MediaPipe Hands solution.
-    implementation 'com.google.mediapipe:hands:latest.release'
-    // Optional: MediaPipe FaceMesh solution.
+    // Optional: MediaPipe Face Detection Solution.
+    implementation 'com.google.mediapipe:facedetection:latest.release'
+    // Optional: MediaPipe Face Mesh Solution.
     implementation 'com.google.mediapipe:facemesh:latest.release'
+    // Optional: MediaPipe Hands Solution.
+    implementation 'com.google.mediapipe:hands:latest.release'
     // MediaPipe deps
-    implementation 'com.google.flogger:flogger:latest.release'
-    implementation 'com.google.flogger:flogger-system-backend:latest.release'
+    implementation 'com.google.flogger:flogger:0.6'
+    implementation 'com.google.flogger:flogger-system-backend:0.6'
     implementation 'com.google.guava:guava:27.0.1-android'
     implementation 'com.google.protobuf:protobuf-java:3.11.4'
     // CameraX core library
@@ -45,7 +47,7 @@ dependencies {
 }
 ```
 
-See the detailed solutions API usage examples for different use cases in the
+See the detailed solution APIs usage examples for different use cases in the
 solution example apps'
 [source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
 If the prebuilt maven packages are not sufficient, building the MediaPipe

docs/getting_started/faq.md

@@ -103,7 +103,7 @@ monotonically increasing timestamps. By convention, realtime calculators and
 graphs use the recording time or the presentation time as the timestamp for each
 packet, with each timestamp representing microseconds since
 `Jan/1/1970:00:00:00`. This allows packets from various sources to be processed
-in a gloablly consistent order.
+in a globally consistent order.
 
 Normally for offline processing, every input packet is processed and processing
 continues as long as necessary. For online processing, it is often necessary to

docs/getting_started/hello_world_ios.md

@@ -113,6 +113,10 @@ bazel to build the iOS application. The content of the
 5.  `Main.storyboard` and `Launch.storyboard`
 6.  `Assets.xcassets` directory.
 
+Note: In newer versions of Xcode, you may see additional files `SceneDelegate.h`
+and `SceneDelegate.m`. Make sure to copy them too and add them to the `BUILD`
+file mentioned below.
+
 Copy these files to a directory named `HelloWorld` to a location that can access
 the MediaPipe source code. For example, the source code of the application that
 we will build in this tutorial is located in
@@ -247,6 +251,12 @@ We need to get frames from the `_cameraSource` into our application
 `MPPInputSourceDelegate`. So our application `ViewController` can be a delegate
 of `_cameraSource`.
 
+Update the interface definition of `ViewController` accordingly:
+
+```
+@interface ViewController () <MPPInputSourceDelegate>
+```
+
 To handle camera setup and process incoming frames, we should use a queue
 different from the main queue. Add the following to the implementation block of
 the `ViewController`:
@@ -288,6 +298,12 @@ utility called `MPPLayerRenderer` to display images on the screen. This utility
 can be used to display `CVPixelBufferRef` objects, which is the type of the
 images provided by `MPPCameraInputSource` to its delegates.
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPLayerRenderer.h"
+```
+
 To display images of the screen, we need to add a new `UIView` object called
 `_liveView` to the `ViewController`.
 
@@ -411,6 +427,12 @@ Objective-C++.
 
 ### Use the graph in `ViewController`
 
+In `ViewController.m`, add the following import line:
+
+```
+#import "mediapipe/objc/MPPGraph.h"
+```
+
 Declare a static constant with the name of the graph, the input stream and the
 output stream:
 
@@ -549,6 +571,12 @@ method to receive packets on this output stream and display them on the screen:
 }
 ```
 
+Update the interface definition of `ViewController` with `MPPGraphDelegate`:
+
+```
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+```
+
 And that is all! Build and run the app on your iOS device. You should see the
 results of running the edge detection graph on a live video feed. Congrats!
 
@@ -560,5 +588,5 @@ appropriate `BUILD` file dependencies for the edge detection graph.
 
 [Bazel]:https://bazel.build/
 [`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt
-[common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common)
-[helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld)
+[common]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common
+[helloworld]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld

docs/getting_started/install.md

@@ -796,7 +796,7 @@ This will use a Docker image that will isolate mediapipe's installation from the
     ```bash
     $ docker run -it --name mediapipe mediapipe:latest
 
-    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazel run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
+    root@bca08b91ff63:/mediapipe# GLOG_logtostderr=1 bazelisk run --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
 
     # Should print:
     # Hello World!

docs/solutions/face_detection.md

@@ -121,12 +121,10 @@ with mp_face_detection.FaceDetection(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_detection.process(image)
 
     # Draw the face detection annotations on the image.
@@ -135,7 +133,8 @@ with mp_face_detection.FaceDetection(
     if results.detections:
       for detection in results.detections:
         mp_drawing.draw_detection(image, detection)
-    cv2.imshow('MediaPipe Face Detection', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Detection', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -200,7 +199,7 @@ const faceDetection = new FaceDetection({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
 }});
 faceDetection.setOptions({
-  modelSelection: 0
+  modelSelection: 0,
   minDetectionConfidence: 0.5
 });
 faceDetection.onResults(onResults);
@@ -216,6 +215,194 @@ camera.start();
 </script>
 ```
 
+### Android Solution API
+
+Please first follow general
+[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
+to add MediaPipe Gradle dependencies, then try the Face Detection Solution API
+in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection)
+following
+[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+and learn more in the usage example below.
+
+*   [staticImageMode](#static_image_mode)
+*   [modelSelection](#model_selection)
+
+#### Camera Input
+
+```java
+// For camera input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Detection Solution.
+CameraInput cameraInput = new CameraInput(this);
+cameraInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      RelativeKeypoint noseTip =
+          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+// The runnable to start camera after the GLSurfaceView is attached.
+glSurfaceView.post(
+    () ->
+        cameraInput.start(
+            this,
+            faceDetection.getGlContext(),
+            CameraInput.CameraFacing.FRONT,
+            glSurfaceView.getWidth(),
+            glSurfaceView.getHeight()));
+```
+
+#### Image Input
+
+```java
+// For reading images from gallery and drawing the output in an ImageView.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(true)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+
+// Connects MediaPipe Face Detection Solution to the user-defined ImageView
+// instance that allows users to have the custom drawing of the output landmarks
+// on it. See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java
+// as an example.
+FaceDetectionResultImageView imageView = new FaceDetectionResultImageView(this);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      int width = faceDetectionResult.inputBitmap().getWidth();
+      int height = faceDetectionResult.inputBitmap().getHeight();
+      RelativeKeypoint noseTip =
+          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
+              noseTip.getX() * width, noseTip.getY() * height));
+      // Request canvas drawing.
+      imageView.setFaceDetectionResult(faceDetectionResult);
+      runOnUiThread(() -> imageView.update());
+    });
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// ActivityResultLauncher to get an image from the gallery as Bitmap.
+ActivityResultLauncher<Intent> imageGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null && result.getResultCode() == RESULT_OK) {
+            Bitmap bitmap = null;
+            try {
+              bitmap =
+                  MediaStore.Images.Media.getBitmap(
+                      this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
+            } catch (IOException e) {
+              Log.e(TAG, "Bitmap reading error:" + e);
+            }
+            if (bitmap != null) {
+              faceDetection.send(bitmap);
+            }
+          }
+        });
+Intent gallery = new Intent(
+    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
+imageGetter.launch(gallery);
+```
+
+#### Video Input
+
+```java
+// For video input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Detection Solution.
+VideoInput videoInput = new VideoInput(this);
+videoInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      RelativeKeypoint noseTip =
+        FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+ActivityResultLauncher<Intent> videoGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null) {
+            if (result.getResultCode() == RESULT_OK) {
+              glSurfaceView.post(
+                  () ->
+                      videoInput.start(
+                          this,
+                          resultIntent.getData(),
+                          faceDetection.getGlContext(),
+                          glSurfaceView.getWidth(),
+                          glSurfaceView.getHeight()));
+            }
+          }
+        });
+Intent gallery =
+    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
+videoGetter.launch(gallery);
+```
+
 ## Example Apps
 
 Please first see general instructions for

docs/solutions/face_mesh.md

@@ -111,6 +111,23 @@ You can find more information about the face landmark model in this
 :------------------------------------------------------------------------: |
 *Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
 
+#### Attention Mesh Model
+
+In addition to the [Face Landmark Model](#face-landmark-model) we provide
+another model that applies
+[attention](https://en.wikipedia.org/wiki/Attention_(machine_learning)) to
+semantically meaningful face regions, and therefore predicting landmarks more
+accurately around lips, eyes and irises, at the expense of more compute. It
+enables applications like AR makeup and AR puppeteering.
+
+The attention mesh model can be selected in the Solution APIs via the
+[refine_landmarks](#refine_landmarks) option. You can also find more information
+about the model in this [paper](https://arxiv.org/abs/2006.10962).
+
+![attention_mesh_architecture.png](../images/attention_mesh_architecture.png) |
+:---------------------------------------------------------------------------: |
+*Fig 3. Attention Mesh: Overview of model architecture.*                      |
+
 ## Face Geometry Module
 
 The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
@@ -145,8 +162,8 @@ be set freely, however for better results it is advised to set them as close to
 the *real physical camera parameters* as possible.
 
 ![face_geometry_metric_3d_space.gif](../images/face_geometry_metric_3d_space.gif) |
-:----------------------------------------------------------------------------: |
-*Fig 3. A visualization of multiple key elements in the Metric 3D space.*      |
+:-------------------------------------------------------------------------------: |
+*Fig 4. A visualization of multiple key elements in the Metric 3D space.*         |
 
 #### Canonical Face Model
 
@@ -210,7 +227,7 @@ The effect renderer is implemented as a MediaPipe
 
 | ![face_geometry_renderer.gif](../images/face_geometry_renderer.gif)     |
 | :---------------------------------------------------------------------: |
-| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
+| *Fig 5. An example of face effects rendered by the Face Geometry Effect Renderer.* |
 
 ## Solution APIs
 
@@ -234,6 +251,12 @@ unrelated, images. Default to `false`.
 
 Maximum number of faces to detect. Default to `1`.
 
+#### refine_landmarks
+
+Whether to further refine the landmark coordinates around the eyes and lips, and
+output additional landmarks around the irises by applying the
+[Attention Mesh Model](#attention-mesh-model). Default to `false`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
@@ -271,6 +294,7 @@ Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
 *   [max_num_faces](#max_num_faces)
+*   [refine_landmarks](#refine_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -287,6 +311,7 @@ drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 with mp_face_mesh.FaceMesh(
     static_image_mode=True,
     max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5) as face_mesh:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
@@ -313,12 +338,21 @@ with mp_face_mesh.FaceMesh(
           landmark_drawing_spec=None,
           connection_drawing_spec=mp_drawing_styles
           .get_default_face_mesh_contours_style())
+      mp_drawing.draw_landmarks(
+          image=annotated_image,
+          landmark_list=face_landmarks,
+          connections=mp_face_mesh.FACEMESH_IRISES,
+          landmark_drawing_spec=None,
+          connection_drawing_spec=mp_drawing_styles
+          .get_default_face_mesh_iris_connections_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
 drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 cap = cv2.VideoCapture(0)
 with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5,
     min_tracking_confidence=0.5) as face_mesh:
   while cap.isOpened():
@@ -328,12 +362,10 @@ with mp_face_mesh.FaceMesh(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_mesh.process(image)
 
     # Draw the face mesh annotations on the image.
@@ -355,7 +387,15 @@ with mp_face_mesh.FaceMesh(
             landmark_drawing_spec=None,
             connection_drawing_spec=mp_drawing_styles
             .get_default_face_mesh_contours_style())
-    cv2.imshow('MediaPipe FaceMesh', image)
+        mp_drawing.draw_landmarks(
+            image=image,
+            landmark_list=face_landmarks,
+            connections=mp_face_mesh.FACEMESH_IRISES,
+            landmark_drawing_spec=None,
+            connection_drawing_spec=mp_drawing_styles
+            .get_default_face_mesh_iris_connections_style())
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Mesh', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -370,6 +410,7 @@ and the following usage example.
 Supported configuration options:
 
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -410,8 +451,10 @@ function onResults(results) {
                      {color: '#C0C0C070', lineWidth: 1});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_IRIS, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_IRIS, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
     }
@@ -424,6 +467,7 @@ const faceMesh = new FaceMesh({locateFile: (file) => {
 }});
 faceMesh.setOptions({
   maxNumFaces: 1,
+  refineLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });
@@ -444,7 +488,7 @@ camera.start();
 
 Please first follow general
 [instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the FaceMash solution API in the
+to add MediaPipe Gradle dependencies, then try the Face Mesh Solution API in the
 companion
 [example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
 following
@@ -455,6 +499,7 @@ Supported configuration options:
 
 *   [staticImageMode](#static_image_mode)
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   runOnGpu: Run the pipeline and the model inference on GPU or CPU.
 
 #### Camera Input
@@ -463,17 +508,18 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Mesh Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -481,18 +527,18 @@ cameraInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -504,7 +550,7 @@ glSurfaceView.post(
     () ->
         cameraInput.start(
             this,
-            facemesh.getGlContext(),
+            faceMesh.getGlContext(),
             CameraInput.CameraFacing.FRONT,
             glSurfaceView.getWidth(),
             glSurfaceView.getHeight()));
@@ -516,17 +562,18 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumFaces(1)                           // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
 
-// Connects MediaPipe FaceMesh to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Face Mesh Solution to the user-defined ImageView instance
+// that allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
 // as an example.
 FaceMeshResultImageView imageView = new FaceMeshResultImageView(this);
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       int width = faceMeshResult.inputBitmap().getWidth();
       int height = faceMeshResult.inputBitmap().getHeight();
@@ -535,14 +582,14 @@ facemesh.setResultListener(
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
+              "MediaPipe Face Mesh nose coordinates (pixel values): x=%f, y=%f",
               noseLandmark.getX() * width, noseLandmark.getY() * height));
       // Request canvas drawing.
       imageView.setFaceMeshResult(faceMeshResult);
       runOnUiThread(() -> imageView.update());
     });
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
 // ActivityResultLauncher to get an image from the gallery as Bitmap.
 ActivityResultLauncher<Intent> imageGetter =
@@ -556,11 +603,12 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
             if (bitmap != null) {
-              facemesh.send(bitmap);
+              faceMesh.send(bitmap);
             }
           }
         });
@@ -575,17 +623,18 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Mesh Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -593,18 +642,18 @@ videoInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -623,7 +672,7 @@ ActivityResultLauncher<Intent> videoGetter =
                       videoInput.start(
                           this,
                           resultIntent.getData(),
-                          facemesh.getGlContext(),
+                          faceMesh.getGlContext(),
                           glSurfaceView.getWidth(),
                           glSurfaceView.getHeight()));
             }

docs/solutions/hands.md

@@ -269,12 +269,10 @@ with mp_hands.Hands(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = hands.process(image)
 
     # Draw the hand annotations on the image.
@@ -288,7 +286,8 @@ with mp_hands.Hands(
             mp_hands.HAND_CONNECTIONS,
             mp_drawing_styles.get_default_hand_landmarks_style(),
             mp_drawing_styles.get_default_hand_connections_style())
-    cv2.imshow('MediaPipe Hands', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Hands', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -372,7 +371,7 @@ camera.start();
 
 Please first follow general
 [instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Hands solution API in the
+to add MediaPipe Gradle dependencies, then try the Hands Solution API in the
 companion
 [example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
 following
@@ -391,14 +390,14 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
+// Initializes a new CameraInput instance and connects it to MediaPipe Hands Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));
@@ -444,13 +443,13 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumHands(1)                        // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 
-// Connects MediaPipe Hands to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Hands Solution to the user-defined ImageView instance that
+// allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
 // as an example.
 HandsResultImageView imageView = new HandsResultImageView(this);
@@ -484,6 +483,7 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
@@ -503,14 +503,14 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
+// Initializes a new VideoInput instance and connects it to MediaPipe Hands Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));

docs/solutions/holistic.md

@@ -147,6 +147,18 @@ If set to `true`, the solution filters pose landmarks across different input
 images to reduce jitter, but ignored if [static_image_mode](#static_image_mode)
 is also set to `true`. Default to `true`.
 
+#### enable_segmentation
+
+If set to `true`, in addition to the pose, face and hand landmarks the solution
+also generates the segmentation mask. Default to `false`.
+
+#### smooth_segmentation
+
+If set to `true`, the solution filters segmentation masks across different input
+images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
+is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
+`true`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
@@ -207,6 +219,15 @@ the camera. The magnitude of `z` uses roughly the same scale as `x`.
 A list of 21 hand landmarks on the right hand, in the same representation as
 [left_hand_landmarks](#left_hand_landmarks).
 
+#### segmentation_mask
+
+The output segmentation mask, predicted only when
+[enable_segmentation](#enable_segmentation) is set to `true`. The mask has the
+same width and height as the input image, and contains values in `[0.0, 1.0]`
+where `1.0` and `0.0` indicate high certainty of a "human" and "background"
+pixel respectively. Please refer to the platform-specific usage examples below
+for usage details.
+
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
@@ -218,6 +239,8 @@ Supported configuration options:
 *   [static_image_mode](#static_image_mode)
 *   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
+*   [enable_segmentation](#enable_segmentation)
+*   [smooth_segmentation](#smooth_segmentation)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -232,7 +255,8 @@ mp_holistic = mp.solutions.holistic
 IMAGE_FILES = []
 with mp_holistic.Holistic(
     static_image_mode=True,
-    model_complexity=2) as holistic:
+    model_complexity=2,
+    enable_segmentation=True) as holistic:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -245,8 +269,16 @@ with mp_holistic.Holistic(
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
       )
-    # Draw pose, left and right hands, and face landmarks on the image.
+
     annotated_image = image.copy()
+    # Draw segmentation on the image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
+    bg_image = np.zeros(image.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    annotated_image = np.where(condition, annotated_image, bg_image)
+    # Draw pose, left and right hands, and face landmarks on the image.
     mp_drawing.draw_landmarks(
         annotated_image,
         results.face_landmarks,
@@ -277,12 +309,10 @@ with mp_holistic.Holistic(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = holistic.process(image)
 
     # Draw landmark annotation on the image.
@@ -301,7 +331,8 @@ with mp_holistic.Holistic(
         mp_holistic.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles
         .get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Holistic', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Holistic', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -317,6 +348,8 @@ Supported configuration options:
 
 *   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
+*   [enableSegmentation](#enable_segmentation)
+*   [smoothSegmentation](#smooth_segmentation)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -349,8 +382,20 @@ const canvasCtx = canvasElement.getContext('2d');
 function onResults(results) {
   canvasCtx.save();
   canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(results.segmentationMask, 0, 0,
+                      canvasElement.width, canvasElement.height);
+
+  // Only overwrite existing pixels.
+  canvasCtx.globalCompositeOperation = 'source-in';
+  canvasCtx.fillStyle = '#00FF00';
+  canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
+
+  // Only overwrite missing pixels.
+  canvasCtx.globalCompositeOperation = 'destination-atop';
   canvasCtx.drawImage(
       results.image, 0, 0, canvasElement.width, canvasElement.height);
+
+  canvasCtx.globalCompositeOperation = 'source-over';
   drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
                  {color: '#00FF00', lineWidth: 4});
   drawLandmarks(canvasCtx, results.poseLandmarks,
@@ -374,6 +419,8 @@ const holistic = new Holistic({locateFile: (file) => {
 holistic.setOptions({
   modelComplexity: 1,
   smoothLandmarks: true,
+  enableSegmentation: true,
+  smoothSegmentation: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });

docs/solutions/models.md

@@ -41,7 +41,10 @@ one over the other.
 *   Face landmark model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   [Model card](https://mediapipe.page.link/facemesh-mc)
+*   Face landmark model w/ attention (aka Attention Mesh):
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite)
+*   [Model card](https://mediapipe.page.link/facemesh-mc),
+    [Model card (w/ attention)](https://mediapipe.page.link/attentionmesh-mc)
 
 ### [Iris](https://google.github.io/mediapipe/solutions/iris)
 

docs/solutions/objectron.md

@@ -338,11 +338,10 @@ with mp_objectron.Objectron(static_image_mode=False,
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Convert the BGR image to RGB.
-    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = objectron.process(image)
 
     # Draw the box landmarks on the image.
@@ -354,7 +353,8 @@ with mp_objectron.Objectron(static_image_mode=False,
               image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
             mp_drawing.draw_axis(image, detected_object.rotation,
                                  detected_object.translation)
-    cv2.imshow('MediaPipe Objectron', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Objectron', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -529,7 +529,7 @@ Example app bounding boxes are rendered with [GlAnimationOverlayCalculator](http
 > ```
 > and then run
 >
-> ```build
+> ```bash
 > bazel run -c opt mediapipe/graphs/object_detection_3d/obj_parser:ObjParser -- input_dir=[INTERMEDIATE_OUTPUT_DIR] output_dir=[OUTPUT_DIR]
 > ```
 > INPUT_DIR should be the folder with initial asset .obj files to be processed,

docs/solutions/pose.md

@@ -88,11 +88,11 @@ from [COCO topology](https://cocodataset.org/#keypoints-2020).
 
 Method                                                                                                | Yoga <br/> [`mAP`] | Yoga <br/> [`[email protected]`] | Dance <br/> [`mAP`] | Dance <br/> [`[email protected]`] | HIIT <br/> [`mAP`] | HIIT <br/> [`[email protected]`]
 ----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------:
-BlazePose.Heavy                                                                                       | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
-BlazePose.Full                                                                                        | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
-BlazePose.Lite                                                                                        | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
-[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
-[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
+BlazePose GHUM Heavy                                                                                  | 68.1               | **96.4**               | 73.0                | **97.2**                | 74.0               | **97.5**
+BlazePose GHUM Full                                                                                   | 62.6               | **95.5**               | 67.4                | **96.3**                | 68.0               | **95.7**
+BlazePose GHUM Lite                                                                                   | 45.0               | **90.2**               | 53.6                | **92.5**                | 53.8               | **93.5**
+[AlphaPose ResNet50](https://github.com/MVIG-SJTU/AlphaPose)                                          | 63.4               | **96.0**               | 57.8                | **95.5**                | 63.4               | **96.0**
+[Apple Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8               | **82.7**               | 36.4                | **91.4**                | 44.5               | **88.6**
 
 ![pose_tracking_pck_chart.png](../images/mobile/pose_tracking_pck_chart.png) |
 :--------------------------------------------------------------------------: |
@@ -101,11 +101,11 @@ BlazePose.Lite
 We designed our models specifically for live perception use cases, so all of
 them work in real-time on the majority of modern devices.
 
-Method          | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
---------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
-BlazePose.Heavy | 53 ms                                                                                        | 38 ms
-BlazePose.Full  | 25 ms                                                                                        | 27 ms
-BlazePose.Lite  | 20 ms                                                                                        | 25 ms
+Method               | Latency <br/> Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency <br/> MacBook Pro (15-inch 2017)
+-------------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------:
+BlazePose GHUM Heavy | 53 ms                                                                                        | 38 ms
+BlazePose GHUM Full  | 25 ms                                                                                        | 27 ms
+BlazePose GHUM Lite  | 20 ms                                                                                        | 25 ms
 
 ## Models
 
@@ -141,7 +141,7 @@ Optionally, MediaPipe Pose can predicts a full-body
 Please find more detail in the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html),
 this [paper](https://arxiv.org/abs/2006.10204),
-[the model card](./models.md#pose) and the [Output](#Output) section below.
+[the model card](./models.md#pose) and the [Output](#output) section below.
 
 ## Solution APIs
 
@@ -237,7 +237,7 @@ pixel respectively. Please refer to the platform-specific usage examples below
 for usage details.
 
 *Fig 6. Example of MediaPipe Pose segmentation mask.* |
-:-----------------------------------------------------------: |
+:---------------------------------------------------: |
 <video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
 
 ### Python Solution API
@@ -281,8 +281,8 @@ with mp_pose.Pose(
       continue
     print(
         f'Nose coordinates: ('
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
-        f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].x * image_width}, '
+        f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].y * image_height})'
     )
 
     annotated_image = image.copy()
@@ -316,12 +316,10 @@ with mp_pose.Pose(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = pose.process(image)
 
     # Draw the pose annotation on the image.
@@ -332,7 +330,8 @@ with mp_pose.Pose(
         results.pose_landmarks,
         mp_pose.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Pose', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Pose', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -369,6 +368,7 @@ Supported configuration options:
   <div class="container">
     <video class="input_video"></video>
     <canvas class="output_canvas" width="1280px" height="720px"></canvas>
+    <div class="landmark-grid-container"></div>
   </div>
 </body>
 </html>

docs/solutions/selfie_segmentation.md

@@ -262,7 +262,7 @@ to visualize its associated subgraphs, please see
     [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1DoeyGzMmWUsjfVgZfGGecrn7GKzYcEAo/view?usp=sharing)
     [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu:selfiesegmentationgpu`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD)
 *   iOS target:
-    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](http:/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
+    [`mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/selfiesegmentationgpu/BUILD)
 
 ### Desktop
 

docs/solutions/solutions.md

@@ -13,6 +13,9 @@ has_toc: false
 {:toc}
 ---
 
+MediaPipe offers open source cross-platform, customizable ML solutions for live
+and streaming media.
+
 <!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
 <!-- Whenever this table is updated, paste a copy to ../external_index.md. -->
 

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -42,4 +42,9 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
 REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
 
+// A calculator to process std::vector<std::vector<Matrix>>.
+typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
+    BeginLoopMatrixVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/flow_limiter_calculator.proto

@@ -30,7 +30,7 @@ message FlowLimiterCalculatorOptions {
   optional int32 max_in_flight = 1 [default = 1];
 
   // The maximum number of frames queued waiting for processing.
-  // The default value limits to 1 frame awaiting processing.
+  // The default value limits to 0 frames awaiting processing.
   optional int32 max_in_queue = 2 [default = 0];
 
   // The maximum time in microseconds to wait for a frame to finish processing.

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -217,6 +217,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
       header->format = video_header.format;
       header->width = video_header.width;
       header->height = video_header.height;
+      header->duration = video_header.duration;
       header->frame_rate = new_frame_rate;
       cc->Outputs().Index(0).SetHeader(Adopt(header.release()));
     } else {

mediapipe/calculators/core/split_vector_calculator.cc

@@ -80,4 +80,7 @@ typedef SplitVectorCalculator<mediapipe::ClassificationList, false>
     SplitClassificationListVectorCalculator;
 REGISTER_CALCULATOR(SplitClassificationListVectorCalculator);
 
+typedef SplitVectorCalculator<uint64_t, false> SplitUint64tVectorCalculator;
+REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -480,8 +480,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
   if (cc->Inputs().HasTag(kRectTag)) {
     const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     // Only use the rect if it is valid.
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
+    if (rect.width() > 0 && rect.height() > 0) {
       x_center = rect.x_center();
       y_center = rect.y_center();
       crop_width = rect.width();

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -337,12 +337,15 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
       !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
     flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
   }
-  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS") &&
-      !cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
-    const auto& image_size =
-        cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
-    output_width_ = image_size.first;
-    output_height_ = image_size.second;
+  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) {
+    if (cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
+      return absl::OkStatus();
+    } else {
+      const auto& image_size =
+          cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
+      output_width_ = image_size.first;
+      output_height_ = image_size.second;
+    }
   }
 
   if (use_gpu_) {
@@ -506,6 +509,14 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
   ComputeOutputDimensions(input_width, input_height, &output_width,
                           &output_height);
 
+  if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
+    const float scale =
+        std::min(static_cast<float>(output_width_) / input_width,
+                 static_cast<float>(output_height_) / input_height);
+    output_width = std::round(input_width * scale);
+    output_height = std::round(input_height * scale);
+  }
+
   if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
     auto padding = absl::make_unique<std::array<float, 4>>();
     ComputeOutputLetterboxPadding(input_width, input_height, output_width,

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -53,7 +53,7 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 // The alpha channel can be set to a single value, or come from an image mask.
 // If the input image has an alpha channel, it will be updated.
 // If the input image doesn't have an alpha channel, one will be added.
-// Adding alpha channel to a Grayscale (single channel) input is not suported.
+// Adding alpha channel to a Grayscale (single channel) input is not supported.
 //
 // Inputs:
 //   One of the following two IMAGE tags:

mediapipe/calculators/tensor/BUILD

@@ -356,6 +356,57 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_to_tensor_calculator_proto",
+    srcs = ["landmarks_to_tensor_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_to_tensor_calculator",
+    srcs = ["landmarks_to_tensor_calculator.cc"],
+    hdrs = ["landmarks_to_tensor_calculator.h"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        ":landmarks_to_tensor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "landmarks_to_tensor_calculator_test",
+    srcs = ["landmarks_to_tensor_calculator_test.cc"],
+    deps = [
+        ":landmarks_to_tensor_calculator",
+        ":landmarks_to_tensor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 mediapipe_proto_library(
     name = "tensors_to_floats_calculator_proto",
     srcs = ["tensors_to_floats_calculator.proto"],

mediapipe/calculators/tensor/inference_calculator.h

@@ -99,13 +99,11 @@ class InferenceCalculator : public NodeIntf {
       kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"};
   static constexpr SideInput<TfLiteModelPtr>::Optional kSideInModel{"MODEL"};
   static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
-  static constexpr SideInput<std::string>::Optional kNnApiDelegateCacheDir{
-      "NNAPI_CACHE_DIR"};
-  static constexpr SideInput<std::string>::Optional kNnApiDelegateModelToken{
-      "NNAPI_MODEL_TOKEN"};
+  static constexpr SideInput<
+      mediapipe::InferenceCalculatorOptions::Delegate>::Optional kDelegate{
+      "DELEGATE"};
   MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel,
-                          kOutTensors, kNnApiDelegateCacheDir,
-                          kNnApiDelegateModelToken);
+                          kOutTensors, kDelegate);
 
  protected:
   using TfLiteDelegatePtr =

mediapipe/calculators/tensor/inference_calculator.proto

@@ -18,6 +18,9 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
+option java_package = "com.google.mediapipe.calculator.proto";
+option java_outer_classname = "InferenceCalculatorProto";
+
 // Full Example:
 //
 // node {

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -50,11 +50,13 @@ int GetXnnpackDefaultNumThreads() {
 // Returns number of threads to configure XNNPACK delegate with.
 // Returns user provided value if specified. Otherwise, tries to choose optimal
 // number of threads depending on the device.
-int GetXnnpackNumThreads(const mediapipe::InferenceCalculatorOptions& opts) {
+int GetXnnpackNumThreads(
+    const bool opts_has_delegate,
+    const mediapipe::InferenceCalculatorOptions::Delegate& opts_delegate) {
   static constexpr int kDefaultNumThreads = -1;
-  if (opts.has_delegate() && opts.delegate().has_xnnpack() &&
-      opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
-    return opts.delegate().xnnpack().num_threads();
+  if (opts_has_delegate && opts_delegate.has_xnnpack() &&
+      opts_delegate.xnnpack().num_threads() != kDefaultNumThreads) {
+    return opts_delegate.xnnpack().num_threads();
   }
   return GetXnnpackDefaultNumThreads();
 }
@@ -73,6 +75,7 @@ class InferenceCalculatorCpuImpl
  private:
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
   Packet<TfLiteModelPtr> model_packet_;
@@ -91,8 +94,7 @@ absl::Status InferenceCalculatorCpuImpl::UpdateContract(
 
 absl::Status InferenceCalculatorCpuImpl::Open(CalculatorContext* cc) {
   MP_RETURN_IF_ERROR(LoadModel(cc));
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
-  return absl::OkStatus();
+  return LoadDelegateAndAllocateTensors(cc);
 }
 
 absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
@@ -156,44 +158,59 @@ absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) {
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 #endif  // __EMSCRIPTEN__
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorCpuImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
-        kTfLiteAffineQuantization);
-
+  RET_CHECK_NE(
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 
 absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
   const auto& calculator_opts =
       cc->Options<mediapipe::InferenceCalculatorOptions>();
-  if (calculator_opts.has_delegate() &&
-      calculator_opts.delegate().has_tflite()) {
+  auto opts_delegate = calculator_opts.delegate();
+  if (!kDelegate(cc).IsEmpty()) {
+    mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
+        kDelegate(cc).Get();
+    CHECK(input_side_packet_delegate.has_tflite() ||
+          input_side_packet_delegate.has_xnnpack() ||
+          input_side_packet_delegate.has_nnapi() ||
+          input_side_packet_delegate.delegate_case() ==
+              mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+        << "inference_calculator_cpu only supports delegate input side packet "
+        << "for TFLite, XNNPack and Nnapi";
+    opts_delegate.MergeFrom(input_side_packet_delegate);
+  }
+  const bool opts_has_delegate =
+      calculator_opts.has_delegate() || !kDelegate(cc).IsEmpty();
+  if (opts_has_delegate && opts_delegate.has_tflite()) {
     // Default tflite inference requeqsted - no need to modify graph.
     return absl::OkStatus();
   }
 
 #if defined(MEDIAPIPE_ANDROID)
-  const bool nnapi_requested = calculator_opts.has_delegate()
-                                   ? calculator_opts.delegate().has_nnapi()
-                                   : calculator_opts.use_nnapi();
+  const bool nnapi_requested = opts_has_delegate ? opts_delegate.has_nnapi()
+                                                 : calculator_opts.use_nnapi();
   if (nnapi_requested) {
     // Attempt to use NNAPI.
     // If not supported, the default CPU delegate will be created and used.
     interpreter_->SetAllowFp16PrecisionForFp32(1);
     tflite::StatefulNnApiDelegate::Options options;
-    const auto& nnapi = calculator_opts.delegate().nnapi();
+    const auto& nnapi = opts_delegate.nnapi();
     // Set up cache_dir and model_token for NNAPI compilation cache.
     options.cache_dir =
         nnapi.has_cache_dir() ? nnapi.cache_dir().c_str() : nullptr;
-    if (!kNnApiDelegateCacheDir(cc).IsEmpty()) {
-      options.cache_dir = kNnApiDelegateCacheDir(cc).Get().c_str();
-    }
     options.model_token =
         nnapi.has_model_token() ? nnapi.model_token().c_str() : nullptr;
-    if (!kNnApiDelegateModelToken(cc).IsEmpty()) {
-      options.model_token = kNnApiDelegateModelToken(cc).Get().c_str();
-    }
     delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options),
                                   [](TfLiteDelegate*) {});
     RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
@@ -205,13 +222,13 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) {
 #if defined(__EMSCRIPTEN__)
   const bool use_xnnpack = true;
 #else
-  const bool use_xnnpack = calculator_opts.has_delegate() &&
-                           calculator_opts.delegate().has_xnnpack();
+  const bool use_xnnpack = opts_has_delegate && opts_delegate.has_xnnpack();
 #endif  // defined(__EMSCRIPTEN__)
 
   if (use_xnnpack) {
     TfLiteXNNPackDelegateOptions xnnpack_opts{};
-    xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts);
+    xnnpack_opts.num_threads =
+        GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
     delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
                                   &TfLiteXNNPackDelegateDelete);
     RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -53,6 +53,7 @@ class InferenceCalculatorGlImpl
   absl::Status WriteKernelsToFile();
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
   absl::Status InitTFLiteGPURunner(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
@@ -94,19 +95,30 @@ absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
 
 absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
   const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
-  use_advanced_gpu_api_ = options.has_delegate() &&
-                          options.delegate().has_gpu() &&
-                          options.delegate().gpu().use_advanced_gpu_api();
-  allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
-  tflite_gpu_runner_api_ = options.delegate().gpu().api();
-  tflite_gpu_runner_usage_ = options.delegate().gpu().usage();
-  use_kernel_caching_ = use_advanced_gpu_api_ &&
-                        options.delegate().gpu().has_cached_kernel_path();
+  mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
+  if (!kDelegate(cc).IsEmpty()) {
+    mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
+        kDelegate(cc).Get();
+    CHECK(input_side_packet_delegate.has_gpu() ||
+          input_side_packet_delegate.delegate_case() ==
+              mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+        << "inference_calculator_gl only supports delegate input side packet "
+        << "for Gpu";
+    delegate.MergeFrom(input_side_packet_delegate);
+  }
+  const bool has_delegate = options.has_delegate() || !kDelegate(cc).IsEmpty();
+  use_advanced_gpu_api_ = has_delegate && delegate.has_gpu() &&
+                          delegate.gpu().use_advanced_gpu_api();
+  allow_precision_loss_ = delegate.gpu().allow_precision_loss();
+  tflite_gpu_runner_api_ = delegate.gpu().api();
+  tflite_gpu_runner_usage_ = delegate.gpu().usage();
+  use_kernel_caching_ =
+      use_advanced_gpu_api_ && delegate.gpu().has_cached_kernel_path();
   use_gpu_delegate_ = !use_advanced_gpu_api_;
 
   if (use_kernel_caching_) {
 #ifdef MEDIAPIPE_ANDROID
-    cached_kernel_filename_ = options.delegate().gpu().cached_kernel_path() +
+    cached_kernel_filename_ = delegate.gpu().cached_kernel_path() +
                               mediapipe::File::Basename(options.model_path()) +
                               ".ker";
 #endif  // MEDIAPIPE_ANDROID
@@ -119,10 +131,11 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
   }
 
   MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this,
-                                                 &cc]() -> ::mediapipe::Status {
-    return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc) : LoadDelegate(cc);
-  }));
+  MP_RETURN_IF_ERROR(
+      gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
+        return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc)
+                                     : LoadDelegateAndAllocateTensors(cc);
+      }));
   return absl::OkStatus();
 }
 
@@ -324,11 +337,19 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 #endif  // __EMSCRIPTEN__
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
-        kTfLiteAffineQuantization);
-
+  RET_CHECK_NE(
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -92,6 +92,7 @@ class InferenceCalculatorMetalImpl
  private:
   absl::Status LoadModel(CalculatorContext* cc);
   absl::Status LoadDelegate(CalculatorContext* cc);
+  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
 
   // TfLite requires us to keep the model alive as long as the interpreter is.
   Packet<TfLiteModelPtr> model_packet_;
@@ -130,8 +131,7 @@ absl::Status InferenceCalculatorMetalImpl::Open(CalculatorContext* cc) {
 
   gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
   RET_CHECK(gpu_helper_);
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
-  return absl::OkStatus();
+  return LoadDelegateAndAllocateTensors(cc);
 }
 
 absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
@@ -212,11 +212,19 @@ absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) {
   interpreter_->SetNumThreads(
       cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread());
 
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorMetalImpl::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+
+  // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
   // TODO: Support quantized tensors.
-  CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type !=
-        kTfLiteAffineQuantization);
-
+  RET_CHECK_NE(
+      interpreter_->tensor(interpreter_->inputs()[0])->quantization.type,
+      kTfLiteAffineQuantization);
   return absl::OkStatus();
 }
 
@@ -236,6 +244,7 @@ absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) {
       TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), &TFLGpuDelegateDelete);
   RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
                kTfLiteOk);
+
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   // Get input image sizes.

mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc

@@ -0,0 +1,101 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
+
+#include <memory>
+
+#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+float GetAttribute(
+    const Landmark& landmark,
+    const LandmarksToTensorCalculatorOptions::Attribute& attribute) {
+  switch (attribute) {
+    case LandmarksToTensorCalculatorOptions::X:
+      return landmark.x();
+    case LandmarksToTensorCalculatorOptions::Y:
+      return landmark.y();
+    case LandmarksToTensorCalculatorOptions::Z:
+      return landmark.z();
+    case LandmarksToTensorCalculatorOptions::VISIBILITY:
+      return landmark.visibility();
+    case LandmarksToTensorCalculatorOptions::PRESENCE:
+      return landmark.presence();
+  }
+}
+
+}  // namespace
+
+class LandmarksToTensorCalculatorImpl
+    : public NodeImpl<LandmarksToTensorCalculator> {
+ public:
+  absl::Status Open(CalculatorContext* cc) override {
+    options_ = cc->Options<LandmarksToTensorCalculatorOptions>();
+    RET_CHECK(options_.attributes_size() > 0)
+        << "At least one attribute must be specified";
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (kInLandmarkList(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+
+    // Get input landmarks.
+    const auto& in_landmarks = *kInLandmarkList(cc);
+
+    // Determine tensor shape.
+    const int n_landmarks = in_landmarks.landmark_size();
+    const int n_attributes = options_.attributes_size();
+    auto tensor_shape = options_.flatten()
+                            ? Tensor::Shape{1, n_landmarks * n_attributes}
+                            : Tensor::Shape{1, n_landmarks, n_attributes};
+
+    // Create empty tesnor.
+    Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
+    auto* buffer = tensor.GetCpuWriteView().buffer<float>();
+
+    // Fill tensor with landmark attributes.
+    for (int i = 0; i < n_landmarks; ++i) {
+      for (int j = 0; j < n_attributes; ++j) {
+        buffer[i * n_attributes + j] =
+            GetAttribute(in_landmarks.landmark(i), options_.attributes(j));
+      }
+    }
+
+    // Return vector with a single tensor.
+    auto result = std::vector<Tensor>();
+    result.push_back(std::move(tensor));
+    kOutTensors(cc).Send(std::move(result));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  LandmarksToTensorCalculatorOptions options_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksToTensorCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h

@@ -0,0 +1,61 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_
+
+#include <memory>
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/tensor.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator for converting landmars into a Tensor.
+//
+// Input:
+//   LANDMARKS - LandmarkList
+//     Landmarks to be converted into a Tensor.
+//
+// Output:
+//   TENSORS - std::vector<Tensor>
+//     Vector containing a single Tensor populated with landmark values.
+//
+// Example:
+// node {
+//   calculator: "LandmarksToTensorCalculator"
+//   input_stream: "LANDMARKS:landmarks"
+//   output_stream: "TENSORS:tensors"
+//   options: {
+//     [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+//       attributes: [X, Y, Z, VISIBILITY, PRESENCE]
+//       # flatten: true
+//     }
+//   }
+// }
+class LandmarksToTensorCalculator : public NodeIntf {
+ public:
+  static constexpr Input<LandmarkList>::Optional kInLandmarkList{"LANDMARKS"};
+  static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
+  MEDIAPIPE_NODE_INTERFACE(LandmarksToTensorCalculator, kInLandmarkList,
+                           kOutTensors);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_LANDMARKS_TO_TENSOR_CALCULATOR_H_

mediapipe/calculators/tensor/landmarks_to_tensor_calculator.proto

@@ -0,0 +1,44 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the LandmarksToTensorCalculator.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message LandmarksToTensorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional LandmarksToTensorCalculatorOptions ext = 394810235;
+  }
+
+  enum Attribute {
+    X = 0;
+    Y = 1;
+    Z = 2;
+    VISIBILITY = 3;
+    PRESENCE = 4;
+  }
+
+  // Subset and order of attributes as they should appear in the output Tensor.
+  // Should contain at least one attribute.
+  repeated Attribute attributes = 1;
+
+  // Collapses all landmark attributes into a one dimensional tensor (i.e.
+  // switches from (n_landmarks, n_attributes) to (n_landmarks * n_attributes)
+  // representation).
+  optional bool flatten = 2 [default = false];
+}

mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc

@@ -0,0 +1,155 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::ParseTextProtoOrDie;
+using Node = ::mediapipe::CalculatorGraphConfig::Node;
+
+void RunLandmarks(mediapipe::CalculatorRunner* runner,
+                  const LandmarkList& landmarks) {
+  runner->MutableInputs()
+      ->Tag("LANDMARKS")
+      .packets.push_back(MakePacket<LandmarkList>(landmarks).At(Timestamp(0)));
+  MP_ASSERT_OK(runner->Run());
+}
+
+const Tensor& GetOutputTensor(mediapipe::CalculatorRunner* runner) {
+  const auto& output_packets = runner->Outputs().Tag("TENSORS").packets;
+  EXPECT_EQ(output_packets.size(), 1);
+
+  const auto& tensors = output_packets[0].Get<std::vector<Tensor>>();
+  EXPECT_EQ(tensors.size(), 1);
+
+  return tensors[0];
+}
+
+void ValidateTensor(const Tensor& tensor,
+                    const std::vector<int>& expected_shape,
+                    const std::vector<float>& expected_values) {
+  EXPECT_EQ(tensor.shape().dims, expected_shape);
+  EXPECT_EQ(tensor.shape().num_elements(), expected_values.size());
+
+  auto* tensor_buffer = tensor.GetCpuReadView().buffer<float>();
+  const std::vector<float> tensor_values(
+      tensor_buffer, tensor_buffer + tensor.shape().num_elements());
+  EXPECT_THAT(tensor_values, testing::ElementsAreArray(expected_values));
+}
+
+TEST(LandmarksToTensorCalculatorTest, AllAttributes) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
+    calculator: "LandmarksToTensorCalculator"
+    input_stream: "LANDMARKS:landmarks"
+    output_stream: "TENSORS:tensors"
+    options: {
+      [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+        attributes: [ X, Y, Z, VISIBILITY, PRESENCE ]
+      }
+    }
+  )pb"));
+
+  LandmarkList landmarks;
+  auto* landmark1 = landmarks.add_landmark();
+  landmark1->set_x(1.0f);
+  landmark1->set_y(2.0f);
+  landmark1->set_z(3.0f);
+  landmark1->set_visibility(4.0f);
+  landmark1->set_presence(5.0f);
+  auto* landmark2 = landmarks.add_landmark();
+  landmark2->set_x(6.0f);
+  landmark2->set_y(7.0f);
+  landmark2->set_z(8.0f);
+  landmark2->set_visibility(9.0f);
+  landmark2->set_presence(10.0f);
+
+  RunLandmarks(&runner, landmarks);
+  const auto& tensor = GetOutputTensor(&runner);
+  ValidateTensor(tensor, /*expected_shape=*/{1, 2, 5}, /*expected_values=*/
+                 {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f});
+}
+
+TEST(LandmarksToTensorCalculatorTest, XYZAttributes) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
+    calculator: "LandmarksToTensorCalculator"
+    input_stream: "LANDMARKS:landmarks"
+    output_stream: "TENSORS:tensors"
+    options: {
+      [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+        attributes: [ X, Y, Z ]
+      }
+    }
+  )pb"));
+
+  LandmarkList landmarks;
+  auto* landmark1 = landmarks.add_landmark();
+  landmark1->set_x(1.0f);
+  landmark1->set_y(2.0f);
+  landmark1->set_z(3.0f);
+  auto* landmark2 = landmarks.add_landmark();
+  landmark2->set_x(6.0f);
+  landmark2->set_y(7.0f);
+  landmark2->set_z(8.0f);
+
+  RunLandmarks(&runner, landmarks);
+  const auto& tensor = GetOutputTensor(&runner);
+  ValidateTensor(tensor, /*expected_shape=*/{1, 2, 3}, /*expected_values=*/
+                 {1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
+}
+
+TEST(LandmarksToTensorCalculatorTest, XYZAttributes_Flatten) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
+    calculator: "LandmarksToTensorCalculator"
+    input_stream: "LANDMARKS:landmarks"
+    output_stream: "TENSORS:tensors"
+    options: {
+      [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+        attributes: [ X, Y, Z ]
+        flatten: true
+      }
+    }
+  )pb"));
+
+  LandmarkList landmarks;
+  auto* landmark1 = landmarks.add_landmark();
+  landmark1->set_x(1.0f);
+  landmark1->set_y(2.0f);
+  landmark1->set_z(3.0f);
+  auto* landmark2 = landmarks.add_landmark();
+  landmark2->set_x(6.0f);
+  landmark2->set_y(7.0f);
+  landmark2->set_z(8.0f);
+
+  RunLandmarks(&runner, landmarks);
+  const auto& tensor = GetOutputTensor(&runner);
+  ValidateTensor(tensor, /*expected_shape=*/{1, 6}, /*expected_values=*/
+                 {1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -670,7 +670,8 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
         detection_boxes[box_offset + 2], detection_boxes[box_offset + 3],
         detection_scores[i], detection_classes[i], options_.flip_vertically());
     const auto& bbox = detection.location_data().relative_bounding_box();
-    if (bbox.width() < 0 || bbox.height() < 0) {
+    if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
+        std::isnan(bbox.height())) {
       // Decoded detection boxes could have negative values for width/height due
       // to model prediction. Filter out those boxes since some downstream
       // calculators may assume non-negative values. (b/171391719)

mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc

@@ -138,7 +138,6 @@ using ::tflite::gpu::gl::GlShader;
 //   }
 // }
 //
-// Currently only OpenGLES 3.1 and CPU backends supported.
 // TODO Refactor and add support for other backends/platforms.
 //
 class TensorsToSegmentationCalculator : public CalculatorBase {

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc

@@ -56,6 +56,8 @@ constexpr char kBboxTag[] = "BBOX";
 constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
 constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
 constexpr char kImageTag[] = "IMAGE";
+constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
+constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
 constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 
 class UnpackMediaSequenceCalculatorTest : public ::testing::Test {

mediapipe/calculators/util/BUILD

@@ -57,6 +57,16 @@ mediapipe_proto_library(
     ],
 )
 
+mediapipe_proto_library(
+    name = "filter_detections_calculator_proto",
+    srcs = ["filter_detections_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 mediapipe_proto_library(
     name = "timed_box_list_id_to_label_calculator_proto",
     srcs = ["timed_box_list_id_to_label_calculator.proto"],
@@ -158,6 +168,21 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "filter_detections_calculator_test",
+    size = "small",
+    srcs = ["filter_detections_calculator_test.cc"],
+    deps = [
+        ":filter_detections_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:message_matchers",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+    ],
+)
+
 cc_library(
     name = "packet_latency_calculator",
     srcs = ["packet_latency_calculator.cc"],
@@ -372,6 +397,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "filter_detections_calculator",
+    srcs = ["filter_detections_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":filter_detections_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "landmarks_to_detection_calculator",
     srcs = ["landmarks_to_detection_calculator.cc"],
@@ -1345,6 +1384,32 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_refinement_calculator_proto",
+    srcs = ["landmarks_refinement_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_refinement_calculator",
+    srcs = ["landmarks_refinement_calculator.cc"],
+    hdrs = ["landmarks_refinement_calculator.h"],
+    deps = [
+        ":landmarks_refinement_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "refine_landmarks_from_heatmap_calculator_test",
     srcs = ["refine_landmarks_from_heatmap_calculator_test.cc"],

mediapipe/calculators/util/filter_detections_calculator.cc

@@ -0,0 +1,81 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iterator>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/util/filter_detections_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+const char kInputDetectionsTag[] = "INPUT_DETECTIONS";
+const char kOutputDetectionsTag[] = "OUTPUT_DETECTIONS";
+
+//
+// Calculator to filter out detections that do not meet the criteria specified
+// in options.
+//
+class FilterDetectionsCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag(kInputDetectionsTag));
+    RET_CHECK(cc->Outputs().HasTag(kOutputDetectionsTag));
+
+    cc->Inputs().Tag(kInputDetectionsTag).Set<std::vector<Detection>>();
+    cc->Outputs().Tag(kOutputDetectionsTag).Set<std::vector<Detection>>();
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    options_ = cc->Options<mediapipe::FilterDetectionsCalculatorOptions>();
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    const auto& input_detections =
+        cc->Inputs().Tag(kInputDetectionsTag).Get<std::vector<Detection>>();
+
+    auto output_detections = absl::make_unique<std::vector<Detection>>();
+
+    for (const Detection& detection : input_detections) {
+      RET_CHECK_GT(detection.score_size(), 0);
+      // Note: only score at index 0 supported.
+      if (detection.score(0) >= options_.min_score()) {
+        output_detections->push_back(detection);
+      }
+    }
+
+    cc->Outputs()
+        .Tag(kOutputDetectionsTag)
+        .Add(output_detections.release(), cc->InputTimestamp());
+
+    return absl::OkStatus();
+  }
+
+ private:
+  mediapipe::FilterDetectionsCalculatorOptions options_;
+};
+
+REGISTER_CALCULATOR(FilterDetectionsCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/filter_detections_calculator.proto

@@ -12,26 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutioncore;
+syntax = "proto2";
 
-import com.google.auto.value.AutoValue;
+package mediapipe;
 
-/**
- * The left, right, bottom, and top boundaries of the visible section on the screen. The boundary
- * values are typically within the range -1.0 and 1.0.
- */
-@AutoValue
-public abstract class ResultGlBoundary {
+import "mediapipe/framework/calculator.proto";
 
-  static ResultGlBoundary create(float left, float right, float bottom, float top) {
-    return new AutoValue_ResultGlBoundary(left, right, bottom, top);
+message FilterDetectionsCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional FilterDetectionsCalculatorOptions ext = 395478132;
   }
 
-  public abstract float left();
-
-  public abstract float right();
-
-  public abstract float bottom();
-
-  public abstract float top();
+  // Detections lower than this score get filtered out.
+  optional float min_score = 1;
 }

mediapipe/calculators/util/filter_detections_calculator_test.cc

@@ -0,0 +1,100 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/message_matchers.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+using ::testing::ElementsAre;
+
+absl::Status RunGraph(std::vector<Detection>& input_detections,
+                      std::vector<Detection>* output_detections) {
+  CalculatorRunner runner(R"pb(
+    calculator: "FilterDetectionsCalculator"
+    input_stream: "INPUT_DETECTIONS:input_detections"
+    output_stream: "OUTPUT_DETECTIONS:output_detections"
+    options {
+      [mediapipe.FilterDetectionsCalculatorOptions.ext] { min_score: 0.5 }
+    }
+  )pb");
+
+  const Timestamp input_timestamp = Timestamp(0);
+  runner.MutableInputs()
+      ->Tag("INPUT_DETECTIONS")
+      .packets.push_back(MakePacket<std::vector<Detection>>(input_detections)
+                             .At(input_timestamp));
+  MP_RETURN_IF_ERROR(runner.Run()) << "Calculator run failed.";
+
+  const std::vector<Packet>& output_packets =
+      runner.Outputs().Tag("OUTPUT_DETECTIONS").packets;
+  RET_CHECK_EQ(output_packets.size(), 1);
+
+  *output_detections = output_packets[0].Get<std::vector<Detection>>();
+  return absl::OkStatus();
+}
+
+TEST(FilterDetectionsCalculatorTest, TestFilterDetections) {
+  std::vector<Detection> input_detections;
+  Detection d1, d2;
+  d1.add_score(0.2);
+  d2.add_score(0.8);
+  input_detections.push_back(d1);
+  input_detections.push_back(d2);
+
+  std::vector<Detection> output_detections;
+  MP_EXPECT_OK(RunGraph(input_detections, &output_detections));
+
+  EXPECT_THAT(output_detections, ElementsAre(mediapipe::EqualsProto(d2)));
+}
+
+TEST(FilterDetectionsCalculatorTest, TestFilterDetectionsMultiple) {
+  std::vector<Detection> input_detections;
+  Detection d1, d2, d3, d4;
+  d1.add_score(0.3);
+  d2.add_score(0.4);
+  d3.add_score(0.5);
+  d4.add_score(0.6);
+  input_detections.push_back(d1);
+  input_detections.push_back(d2);
+  input_detections.push_back(d3);
+  input_detections.push_back(d4);
+
+  std::vector<Detection> output_detections;
+  MP_EXPECT_OK(RunGraph(input_detections, &output_detections));
+
+  EXPECT_THAT(output_detections, ElementsAre(mediapipe::EqualsProto(d3),
+                                             mediapipe::EqualsProto(d4)));
+}
+
+TEST(FilterDetectionsCalculatorTest, TestFilterDetectionsEmpty) {
+  std::vector<Detection> input_detections;
+
+  std::vector<Detection> output_detections;
+  MP_EXPECT_OK(RunGraph(input_detections, &output_detections));
+
+  EXPECT_EQ(output_detections.size(), 0);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/util/labels_to_render_data_calculator.cc

@@ -175,7 +175,8 @@ absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) {
 
     auto* text = label_annotation->mutable_text();
     std::string display_text = labels[i];
-    if (cc->Inputs().HasTag(kScoresTag)) {
+    if (cc->Inputs().HasTag(kScoresTag) ||
+        options_.display_classification_score()) {
       absl::StrAppend(&display_text, ":", scores[i]);
     }
     text->set_display_text(display_text);

mediapipe/calculators/util/labels_to_render_data_calculator.proto

@@ -62,4 +62,7 @@ message LabelsToRenderDataCalculatorOptions {
 
   // Uses Classification.display_name field instead of Classification.label.
   optional bool use_display_name = 9 [default = false];
+
+  // Displays Classification score if enabled.
+  optional bool display_classification_score = 10 [default = false];
 }

mediapipe/calculators/util/landmarks_refinement_calculator.cc

@@ -0,0 +1,197 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/landmarks_refinement_calculator.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/util/landmarks_refinement_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+namespace api2 {
+
+namespace {
+
+absl::StatusOr<int> GetNumberOfRefinedLandmarks(
+    const proto_ns::RepeatedPtrField<
+        LandmarksRefinementCalculatorOptions::Refinement>& refinements) {
+  // Gather all used indexes.
+  std::set<int> idxs;
+  for (int i = 0; i < refinements.size(); ++i) {
+    const auto& refinement = refinements.Get(i);
+    for (int i = 0; i < refinement.indexes_mapping_size(); ++i) {
+      idxs.insert(refinement.indexes_mapping(i));
+    }
+  }
+
+  // Check that indxes start with 0 and there is no gaps between min and max
+  // indexes.
+  RET_CHECK(!idxs.empty())
+      << "There should be at least one landmark in indexes mapping";
+  int idxs_min = *idxs.begin();
+  int idxs_max = *idxs.rbegin();
+  int n_idxs = idxs.size();
+  RET_CHECK_EQ(idxs_min, 0)
+      << "Indexes are expected to start with 0 instead of " << idxs_min;
+  RET_CHECK_EQ(idxs_max, n_idxs - 1)
+      << "Indexes should have no gaps but " << idxs_max - n_idxs + 1
+      << " indexes are missing";
+
+  return n_idxs;
+}
+
+void RefineXY(const proto_ns::RepeatedField<int>& indexes_mapping,
+              const NormalizedLandmarkList& landmarks,
+              NormalizedLandmarkList* refined_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+    auto* refined_landmark =
+        refined_landmarks->mutable_landmark(indexes_mapping.Get(i));
+    refined_landmark->set_x(landmark.x());
+    refined_landmark->set_y(landmark.y());
+  }
+}
+
+float GetZAverage(const NormalizedLandmarkList& landmarks,
+                  const proto_ns::RepeatedField<int>& indexes) {
+  double z_sum = 0;
+  for (int i = 0; i < indexes.size(); ++i) {
+    z_sum += landmarks.landmark(indexes.Get(i)).z();
+  }
+  return z_sum / indexes.size();
+}
+
+void RefineZ(
+    const proto_ns::RepeatedField<int>& indexes_mapping,
+    const LandmarksRefinementCalculatorOptions::ZRefinement& z_refinement,
+    const NormalizedLandmarkList& landmarks,
+    NormalizedLandmarkList* refined_landmarks) {
+  if (z_refinement.has_none()) {
+    // Do nothing and keep Z that is already in refined landmarks.
+  } else if (z_refinement.has_copy()) {
+    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+      refined_landmarks->mutable_landmark(indexes_mapping.Get(i))
+          ->set_z(landmarks.landmark(i).z());
+    }
+  } else if (z_refinement.has_assign_average()) {
+    const float z_average =
+        GetZAverage(*refined_landmarks,
+                    z_refinement.assign_average().indexes_for_average());
+    for (int i = 0; i < indexes_mapping.size(); ++i) {
+      refined_landmarks->mutable_landmark(indexes_mapping.Get(i))
+          ->set_z(z_average);
+    }
+  } else {
+    CHECK(false) << "Z refinement is either not specified or not supported";
+  }
+}
+
+}  // namespace
+
+class LandmarksRefinementCalculatorImpl
+    : public NodeImpl<LandmarksRefinementCalculator> {
+  absl::Status Open(CalculatorContext* cc) override {
+    options_ = cc->Options<LandmarksRefinementCalculatorOptions>();
+
+    // Validate refinements.
+    for (int i = 0; i < options_.refinement_size(); ++i) {
+      const auto& refinement = options_.refinement(i);
+      RET_CHECK_GT(refinement.indexes_mapping_size(), 0)
+          << "Refinement " << i << " has no indexes mapping";
+      RET_CHECK(refinement.has_z_refinement())
+          << "Refinement " << i << " has no Z refinement specified";
+      RET_CHECK(refinement.z_refinement().has_none() ^
+                refinement.z_refinement().has_copy() ^
+                refinement.z_refinement().has_assign_average())
+          << "Exactly one Z refinement should be specified";
+
+      const auto z_refinement = refinement.z_refinement();
+      if (z_refinement.has_assign_average()) {
+        RET_CHECK_GT(z_refinement.assign_average().indexes_for_average_size(),
+                     0)
+            << "When using assign average Z refinement at least one index for "
+               "averagin should be specified";
+      }
+    }
+
+    // Validate indexes mapping and get total number of refined landmarks.
+    ASSIGN_OR_RETURN(n_refined_landmarks_,
+                     GetNumberOfRefinedLandmarks(options_.refinement()));
+
+    // Validate that number of refinements and landmark streams is the same.
+    RET_CHECK_EQ(kLandmarks(cc).Count(), options_.refinement_size())
+        << "There are " << options_.refinement_size() << " refinements while "
+        << kLandmarks(cc).Count() << " landmark streams";
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // If any of the refinement landmarks is missing - refinement won't happen.
+    for (const auto& landmarks_stream : kLandmarks(cc)) {
+      if (landmarks_stream.IsEmpty()) {
+        return absl::OkStatus();
+      }
+    }
+
+    // Initialize refined landmarks list.
+    auto refined_landmarks = absl::make_unique<NormalizedLandmarkList>();
+    for (int i = 0; i < n_refined_landmarks_; ++i) {
+      refined_landmarks->add_landmark();
+    }
+
+    // Apply input landmarks to outpu refined landmarks in provided order.
+    for (int i = 0; i < kLandmarks(cc).Count(); ++i) {
+      const auto& landmarks = kLandmarks(cc)[i].Get();
+      const auto& refinement = options_.refinement(i);
+
+      // Check number of landmarks in mapping and stream are the same.
+      RET_CHECK_EQ(landmarks.landmark_size(), refinement.indexes_mapping_size())
+          << "There are " << landmarks.landmark_size()
+          << " refinement landmarks while mapping has "
+          << refinement.indexes_mapping_size();
+
+      // Refine X and Y.
+      RefineXY(refinement.indexes_mapping(), landmarks,
+               refined_landmarks.get());
+
+      // Refine Z.
+      RefineZ(refinement.indexes_mapping(), refinement.z_refinement(),
+              landmarks, refined_landmarks.get());
+
+      // Visibility and presence are not currently refined and are left as `0`.
+    }
+
+    kRefinedLandmarks(cc).Send(std::move(refined_landmarks));
+    return absl::OkStatus();
+  }
+
+ private:
+  LandmarksRefinementCalculatorOptions options_;
+  int n_refined_landmarks_ = 0;
+};
+
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksRefinementCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_refinement_calculator.h

@@ -0,0 +1,85 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+
+namespace api2 {
+
+// A calculator to refine one set of landmarks with another.
+//
+// Inputs:
+//   LANDMARKS: Multiple NormalizedLandmarkList to use for
+//     refinement. They will be applied to the resulting REFINED_LANDMARKS in
+//     the provided order. Each list should be non empty and contain the same
+//     amount of landmarks as indexes in mapping. Number of lists should be the
+//     same as number of refinements in options.
+//
+// Outputs:
+//   REFINED_LANDMARKS: A NormalizedLandmarkList with refined landmarks. Number
+//     of produced landmarks is equal to to the maximum index mapping number in
+//     calculator options (calculator verifies that there are no gaps in the
+//     mapping).
+//
+// Examples config:
+//   node {
+//     calculator: "LandmarksRefinementCalculator"
+//     input_stream: "LANDMARKS:0:mesh_landmarks"
+//     input_stream: "LANDMARKS:1:lips_landmarks"
+//     input_stream: "LANDMARKS:2:left_eye_landmarks"
+//     input_stream: "LANDMARKS:3:right_eye_landmarks"
+//     output_stream: "REFINED_LANDMARKS:landmarks"
+//     options: {
+//       [mediapipe.LandmarksRefinementCalculatorOptions.ext] {
+//         refinement: {
+//           indexes_mapping: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+//           z_refinement: { copy {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [0, 1, 2, 3]
+//           z_refinement: { none {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [4, 5]
+//           z_refinement: { none {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [6, 7]
+//           z_refinement: { none {} }
+//         }
+//       }
+//     }
+//   }
+//
+class LandmarksRefinementCalculator : public NodeIntf {
+ public:
+  static constexpr Input<::mediapipe::NormalizedLandmarkList>::Multiple
+      kLandmarks{"LANDMARKS"};
+  static constexpr Output<::mediapipe::NormalizedLandmarkList>
+      kRefinedLandmarks{"REFINED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(LandmarksRefinementCalculator, kLandmarks,
+                           kRefinedLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_

mediapipe/calculators/util/landmarks_refinement_calculator.proto

@@ -0,0 +1,71 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message LandmarksRefinementCalculatorOptions {
+  extend CalculatorOptions {
+    optional LandmarksRefinementCalculatorOptions ext = 381914658;
+  }
+
+  // Do nothing and keep those Z that are already present in the resulting set
+  // of landmarks.
+  message ZRefinementNone {}
+
+  // Simply copy Z values from the given set of landmarks to the resulting set
+  // of landmarks.
+  message ZRefinementCopy {}
+
+  // Calculate average of the specified set of landmarks in the resulting set
+  // and use it as Z for all given landmarks when assigning their values to the
+  // resulting set of landmarks.
+  message ZRefinementAssignAverage {
+    // Indexes of the resulting landmarks to use for average. Should be non
+    // empty.
+    repeated int32 indexes_for_average = 1;
+  }
+
+  // Specifies the set of instructions on assigning z value from the given set
+  // of landmarks to the resulting set of landmarks.
+  message ZRefinement {
+    // Exactly one Z refinement option should be specified.
+    oneof z_refinement_options {
+      ZRefinementNone none = 1;
+      ZRefinementCopy copy = 2;
+      ZRefinementAssignAverage assign_average = 3;
+    }
+  }
+
+  // Specifies the set of instructions of assigning values to the resulting set
+  // of landmarks.
+  message Refinement {
+    // Maps indexes of the given set of landmarks to indexes of the resulting
+    // set of landmarks. Should be non empty and contain the same amount of
+    // indexes as landmarks in the corresponding input stream.
+    repeated int32 indexes_mapping = 1;
+
+    // Z refinement instructions.
+    optional ZRefinement z_refinement = 2;
+  }
+
+  // Refinement instructions for every landmarks input stream. Applied in the
+  // same order as defined. Should be the same amount of refinements as landmark
+  // input streams in the calculator. Union of index mappings should start with
+  // 0 and cover a contineous range.
+  repeated Refinement refinement = 1;
+}

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -86,11 +86,11 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
 }
 
 template <class LandmarkType>
-bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark,
-                                  bool utilize_visibility,
-                                  float visibility_threshold,
-                                  bool utilize_presence,
-                                  float presence_threshold) {
+bool IsLandmarkVisibleAndPresent(const LandmarkType& landmark,
+                                 bool utilize_visibility,
+                                 float visibility_threshold,
+                                 bool utilize_presence,
+                                 float presence_threshold) {
   if (utilize_visibility && landmark.has_visibility() &&
       landmark.visibility() < visibility_threshold) {
     return false;
@@ -153,12 +153,16 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
                              const Color& max_depth_line_color,
                              RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
+    if (landmark_connections[i] >= landmarks.landmark_size() ||
+        landmark_connections[i + 1] >= landmarks.landmark_size()) {
+      continue;
+    }
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
+    if (!IsLandmarkVisibleAndPresent<LandmarkType>(
             ld0, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold) ||
-        !IsLandmarkVisibileAndPresent<LandmarkType>(
+        !IsLandmarkVisibleAndPresent<LandmarkType>(
             ld1, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold)) {
       continue;
@@ -196,12 +200,16 @@ void AddConnections(const LandmarkListType& landmarks,
                     const Color& connection_color, float thickness,
                     bool normalized, RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
+    if (landmark_connections[i] >= landmarks.landmark_size() ||
+        landmark_connections[i + 1] >= landmarks.landmark_size()) {
+      continue;
+    }
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
+    if (!IsLandmarkVisibleAndPresent<LandmarkType>(
             ld0, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold) ||
-        !IsLandmarkVisibileAndPresent<LandmarkType>(
+        !IsLandmarkVisibleAndPresent<LandmarkType>(
             ld1, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold)) {
       continue;
@@ -317,7 +325,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibileAndPresent<Landmark>(
+      if (!IsLandmarkVisibleAndPresent<Landmark>(
               landmark, options_.utilize_visibility(),
               options_.visibility_threshold(), options_.utilize_presence(),
               options_.presence_threshold())) {
@@ -363,7 +371,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibileAndPresent<NormalizedLandmark>(
+      if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
               landmark, options_.utilize_visibility(),
               options_.visibility_threshold(), options_.utilize_presence(),
               options_.presence_threshold())) {

mediapipe/calculators/util/rect_transformation_calculator.cc

@@ -36,7 +36,7 @@ inline float NormalizeRadians(float angle) {
 }  // namespace
 
 // Performs geometric transformation to the input Rect or NormalizedRect,
-// correpsonding to input stream RECT or NORM_RECT respectively. When the input
+// corresponding to input stream RECT or NORM_RECT respectively. When the input
 // is NORM_RECT, an addition input stream IMAGE_SIZE is required, which is a
 // std::pair<int, int> representing the image width and height.
 //

mediapipe/examples/android/solutions/create_win_symlinks.bat

@@ -12,5 +12,12 @@ cd /d %~dp0
 cd facemesh\src\main
 rm res
 mklink /d res ..\..\..\res
+
+@rem for face detection example app.
+cd /d %~dp0
+cd facedetection\src\main
+rm res
+mklink /d res ..\..\..\res
+
 dir
 pause

mediapipe/examples/android/solutions/facedetection/build.gradle

@@ -0,0 +1,51 @@
+plugins {
+    id 'com.android.application'
+}
+
+android {
+    compileSdkVersion 30
+    buildToolsVersion "30.0.3"
+
+    defaultConfig {
+        applicationId "com.google.mediapipe.apps.facedetection"
+        minSdkVersion 21
+        targetSdkVersion 30
+        versionCode 1
+        versionName "1.0"
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+}
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar'])
+    implementation 'androidx.appcompat:appcompat:1.3.0'
+    implementation 'com.google.android.material:material:1.3.0'
+    implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
+    implementation 'androidx.exifinterface:exifinterface:1.3.3'
+    testImplementation 'junit:junit:4.+'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.2'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
+    // MediaPipe Face Detection Solution components.
+    implementation 'com.google.mediapipe:solution-core:latest.release'
+    implementation 'com.google.mediapipe:facedetection:latest.release'
+    // MediaPipe deps
+    implementation 'com.google.flogger:flogger:0.6'
+    implementation 'com.google.flogger:flogger-system-backend:0.6'
+    implementation 'com.google.guava:guava:27.0.1-android'
+    implementation 'com.google.protobuf:protobuf-java:3.11.4'
+    // CameraX core library
+    def camerax_version = "1.0.0-beta10"
+    implementation "androidx.camera:camera-core:$camerax_version"
+    implementation "androidx.camera:camera-camera2:$camerax_version"
+    implementation "androidx.camera:camera-lifecycle:$camerax_version"
+}

mediapipe/examples/android/solutions/facedetection/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.examples.facedetection">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="30" />
+
+  <!-- For loading images from gallery -->
+  <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+
+  <application
+      android:allowBackup="true"
+      android:icon="@mipmap/ic_launcher"
+      android:label="MediaPipe Face Detection"
+      android:roundIcon="@mipmap/ic_launcher_round"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+    <activity android:name=".MainActivity"
+              android:screenOrientation="portrait">
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN" />
+
+        <category android:name="android.intent.category.LAUNCHER" />
+      </intent-filter>
+    </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/solutions/facedetection/src/main/BUILD

@@ -0,0 +1,46 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:private"])
+
+android_binary(
+    name = "facedetection",
+    srcs = glob(["**/*.java"]),
+    custom_package = "com.google.mediapipe.examples.facedetection",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.examples.facedetection",
+    },
+    multidex = "native",
+    resource_files = ["//mediapipe/examples/android/solutions:resource_files"],
+    deps = [
+        "//mediapipe/framework/formats:detection_java_proto_lite",
+        "//mediapipe/framework/formats:location_data_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/solutioncore:camera_input",
+        "//mediapipe/java/com/google/mediapipe/solutioncore:mediapipe_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/solutioncore:solution_rendering",
+        "//mediapipe/java/com/google/mediapipe/solutioncore:video_input",
+        "//mediapipe/java/com/google/mediapipe/solutions/facedetection",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:opencv",
+        "@maven//:androidx_activity_activity",
+        "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_exifinterface_exifinterface",
+        "@maven//:androidx_fragment_fragment",
+        "@maven//:com_google_guava_guava",
+    ],
+)

mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java

@@ -0,0 +1,146 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.examples.facedetection;
+
+import android.opengl.GLES20;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
+import com.google.mediapipe.solutioncore.ResultGlRenderer;
+import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
+import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+
+/** A custom implementation of {@link ResultGlRenderer} to render {@link FaceDetectionResult}. */
+public class FaceDetectionResultGlRenderer implements ResultGlRenderer<FaceDetectionResult> {
+  private static final String TAG = "FaceDetectionResultGlRenderer";
+
+  private static final float[] KEYPOINT_COLOR = new float[] {1f, 0f, 0f, 1f};
+  private static final float KEYPOINT_SIZE = 16f;
+  private static final float[] BBOX_COLOR = new float[] {0f, 1f, 0f, 1f};
+  private static final int BBOX_THICKNESS = 8;
+  private static final String VERTEX_SHADER =
+      "uniform mat4 uProjectionMatrix;\n"
+          + "uniform float uPointSize;\n"
+          + "attribute vec4 vPosition;\n"
+          + "void main() {\n"
+          + "  gl_Position = uProjectionMatrix * vPosition;\n"
+          + "  gl_PointSize = uPointSize;"
+          + "}";
+  private static final String FRAGMENT_SHADER =
+      "precision mediump float;\n"
+          + "uniform vec4 uColor;\n"
+          + "void main() {\n"
+          + "  gl_FragColor = uColor;\n"
+          + "}";
+  private int program;
+  private int positionHandle;
+  private int pointSizeHandle;
+  private int projectionMatrixHandle;
+  private int colorHandle;
+
+  private int loadShader(int type, String shaderCode) {
+    int shader = GLES20.glCreateShader(type);
+    GLES20.glShaderSource(shader, shaderCode);
+    GLES20.glCompileShader(shader);
+    return shader;
+  }
+
+  @Override
+  public void setupRendering() {
+    program = GLES20.glCreateProgram();
+    int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER);
+    int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
+    GLES20.glAttachShader(program, vertexShader);
+    GLES20.glAttachShader(program, fragmentShader);
+    GLES20.glLinkProgram(program);
+    positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
+    pointSizeHandle = GLES20.glGetUniformLocation(program, "uPointSize");
+    projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
+    colorHandle = GLES20.glGetUniformLocation(program, "uColor");
+  }
+
+  @Override
+  public void renderResult(FaceDetectionResult result, float[] projectionMatrix) {
+    if (result == null) {
+      return;
+    }
+    GLES20.glUseProgram(program);
+    GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
+    GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
+    int numDetectedFaces = result.multiFaceDetections().size();
+    for (int i = 0; i < numDetectedFaces; ++i) {
+      drawDetection(result.multiFaceDetections().get(i));
+    }
+  }
+
+  /**
+   * Deletes the shader program.
+   *
+   * <p>This is only necessary if one wants to release the program while keeping the context around.
+   */
+  public void release() {
+    GLES20.glDeleteProgram(program);
+  }
+
+  private void drawDetection(Detection detection) {
+    if (!detection.hasLocationData()) {
+      return;
+    }
+    // Draw keypoints.
+    float[] points = new float[FaceKeypoint.NUM_KEY_POINTS * 2];
+    for (int i = 0; i < FaceKeypoint.NUM_KEY_POINTS; ++i) {
+      points[2 * i] = detection.getLocationData().getRelativeKeypoints(i).getX();
+      points[2 * i + 1] = detection.getLocationData().getRelativeKeypoints(i).getY();
+    }
+    GLES20.glUniform4fv(colorHandle, 1, KEYPOINT_COLOR, 0);
+    FloatBuffer vertexBuffer =
+        ByteBuffer.allocateDirect(points.length * 4)
+            .order(ByteOrder.nativeOrder())
+            .asFloatBuffer()
+            .put(points);
+    vertexBuffer.position(0);
+    GLES20.glEnableVertexAttribArray(positionHandle);
+    GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
+    GLES20.glDrawArrays(GLES20.GL_POINTS, 0, FaceKeypoint.NUM_KEY_POINTS);
+    if (!detection.getLocationData().hasRelativeBoundingBox()) {
+      return;
+    }
+    // Draw bounding box.
+    float left = detection.getLocationData().getRelativeBoundingBox().getXmin();
+    float top = detection.getLocationData().getRelativeBoundingBox().getYmin();
+    float right = left + detection.getLocationData().getRelativeBoundingBox().getWidth();
+    float bottom = top + detection.getLocationData().getRelativeBoundingBox().getHeight();
+    drawLine(top, left, top, right);
+    drawLine(bottom, left, bottom, right);
+    drawLine(top, left, bottom, left);
+    drawLine(top, right, bottom, right);
+  }
+
+  private void drawLine(float y1, float x1, float y2, float x2) {
+    GLES20.glUniform4fv(colorHandle, 1, BBOX_COLOR, 0);
+    GLES20.glLineWidth(BBOX_THICKNESS);
+    float[] vertex = {x1, y1, x2, y2};
+    FloatBuffer vertexBuffer =
+        ByteBuffer.allocateDirect(vertex.length * 4)
+            .order(ByteOrder.nativeOrder())
+            .asFloatBuffer()
+            .put(vertex);
+    vertexBuffer.position(0);
+    GLES20.glEnableVertexAttribArray(positionHandle);
+    GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
+    GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
+  }
+}

mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java

@@ -0,0 +1,108 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.examples.facedetection;
+
+import static java.lang.Math.min;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import androidx.appcompat.widget.AppCompatImageView;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
+import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
+import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
+
+/** An ImageView implementation for displaying {@link FaceDetectionResult}. */
+public class FaceDetectionResultImageView extends AppCompatImageView {
+  private static final String TAG = "FaceDetectionResultImageView";
+
+  private static final int KEYPOINT_COLOR = Color.RED;
+  private static final int KEYPOINT_RADIUS = 15;
+  private static final int BBOX_COLOR = Color.GREEN;
+  private static final int BBOX_THICKNESS = 10;
+  private Bitmap latest;
+
+  public FaceDetectionResultImageView(Context context) {
+    super(context);
+    setScaleType(AppCompatImageView.ScaleType.FIT_CENTER);
+  }
+
+  /**
+   * Sets a {@link FaceDetectionResult} to render.
+   *
+   * @param result a {@link FaceDetectionResult} object that contains the solution outputs and the
+   *     input {@link Bitmap}.
+   */
+  public void setFaceDetectionResult(FaceDetectionResult result) {
+    if (result == null) {
+      return;
+    }
+    Bitmap bmInput = result.inputBitmap();
+    int width = bmInput.getWidth();
+    int height = bmInput.getHeight();
+    latest = Bitmap.createBitmap(width, height, bmInput.getConfig());
+    Canvas canvas = new Canvas(latest);
+
+    canvas.drawBitmap(bmInput, new Matrix(), null);
+    int numDetectedFaces = result.multiFaceDetections().size();
+    for (int i = 0; i < numDetectedFaces; ++i) {
+      drawDetectionOnCanvas(result.multiFaceDetections().get(i), canvas, width, height);
+    }
+  }
+
+  /** Updates the image view with the latest {@link FaceDetectionResult}. */
+  public void update() {
+    postInvalidate();
+    if (latest != null) {
+      setImageBitmap(latest);
+    }
+  }
+
+  private void drawDetectionOnCanvas(Detection detection, Canvas canvas, int width, int height) {
+    if (!detection.hasLocationData()) {
+      return;
+    }
+    // Draw keypoints.
+    Paint keypointPaint = new Paint();
+    keypointPaint.setColor(KEYPOINT_COLOR);
+    for (int i = 0; i < FaceKeypoint.NUM_KEY_POINTS; ++i) {
+      int xPixel =
+          min(
+              (int) (detection.getLocationData().getRelativeKeypoints(i).getX() * width),
+              width - 1);
+      int yPixel =
+          min(
+              (int) (detection.getLocationData().getRelativeKeypoints(i).getY() * height),
+              height - 1);
+      canvas.drawCircle(xPixel, yPixel, KEYPOINT_RADIUS, keypointPaint);
+    }
+    if (!detection.getLocationData().hasRelativeBoundingBox()) {
+      return;
+    }
+    // Draw bounding box.
+    Paint bboxPaint = new Paint();
+    bboxPaint.setColor(BBOX_COLOR);
+    bboxPaint.setStyle(Paint.Style.STROKE);
+    bboxPaint.setStrokeWidth(BBOX_THICKNESS);
+    float left = detection.getLocationData().getRelativeBoundingBox().getXmin() * width;
+    float top = detection.getLocationData().getRelativeBoundingBox().getYmin() * height;
+    float right = left + detection.getLocationData().getRelativeBoundingBox().getWidth() * width;
+    float bottom = top + detection.getLocationData().getRelativeBoundingBox().getHeight() * height;
+    canvas.drawRect(left, top, right, bottom, bboxPaint);
+  }
+}

mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/MainActivity.java

@@ -0,0 +1,341 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.examples.facedetection;
+
+import android.content.Intent;
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.os.Bundle;
+import android.provider.MediaStore;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Log;
+import android.view.View;
+import android.widget.Button;
+import android.widget.FrameLayout;
+import androidx.activity.result.ActivityResultLauncher;
+import androidx.activity.result.contract.ActivityResultContracts;
+import androidx.exifinterface.media.ExifInterface;
+// ContentResolver dependency
+import com.google.mediapipe.formats.proto.LocationDataProto.LocationData.RelativeKeypoint;
+import com.google.mediapipe.solutioncore.CameraInput;
+import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
+import com.google.mediapipe.solutioncore.VideoInput;
+import com.google.mediapipe.solutions.facedetection.FaceDetection;
+import com.google.mediapipe.solutions.facedetection.FaceDetectionOptions;
+import com.google.mediapipe.solutions.facedetection.FaceDetectionResult;
+import com.google.mediapipe.solutions.facedetection.FaceKeypoint;
+import java.io.IOException;
+import java.io.InputStream;
+
+/** Main activity of MediaPipe Face Detection app. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private FaceDetection faceDetection;
+
+  private enum InputSource {
+    UNKNOWN,
+    IMAGE,
+    VIDEO,
+    CAMERA,
+  }
+  private InputSource inputSource = InputSource.UNKNOWN;
+
+  // Image demo UI and image loader components.
+  private ActivityResultLauncher<Intent> imageGetter;
+  private FaceDetectionResultImageView imageView;
+  // Video demo UI and video loader components.
+  private VideoInput videoInput;
+  private ActivityResultLauncher<Intent> videoGetter;
+  // Live camera demo UI and camera components.
+  private CameraInput cameraInput;
+
+  private SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+    setupStaticImageDemoUiComponents();
+    setupVideoDemoUiComponents();
+    setupLiveDemoUiComponents();
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    if (inputSource == InputSource.CAMERA) {
+      // Restarts the camera and the opengl surface rendering.
+      cameraInput = new CameraInput(this);
+      cameraInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
+      glSurfaceView.post(this::startCamera);
+      glSurfaceView.setVisibility(View.VISIBLE);
+    } else if (inputSource == InputSource.VIDEO) {
+      videoInput.resume();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    if (inputSource == InputSource.CAMERA) {
+      glSurfaceView.setVisibility(View.GONE);
+      cameraInput.close();
+    } else if (inputSource == InputSource.VIDEO) {
+      videoInput.pause();
+    }
+  }
+
+  /** Sets up the UI components for the static image demo. */
+  private void setupStaticImageDemoUiComponents() {
+    // The Intent to access gallery and read images as bitmap.
+    imageGetter =
+        registerForActivityResult(
+            new ActivityResultContracts.StartActivityForResult(),
+            result -> {
+              Intent resultIntent = result.getData();
+              if (resultIntent != null) {
+                if (result.getResultCode() == RESULT_OK) {
+                  Bitmap bitmap = null;
+                  try {
+                    bitmap =
+                        MediaStore.Images.Media.getBitmap(
+                            this.getContentResolver(), resultIntent.getData());
+                  } catch (IOException e) {
+                    Log.e(TAG, "Bitmap reading error:" + e);
+                  }
+                  try {
+                    InputStream imageData =
+                        this.getContentResolver().openInputStream(resultIntent.getData());
+                    int orientation =
+                        new ExifInterface(imageData)
+                            .getAttributeInt(
+                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
+                      Matrix matrix = new Matrix();
+                      switch (orientation) {
+                        case ExifInterface.ORIENTATION_ROTATE_90:
+                          matrix.postRotate(90);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_180:
+                          matrix.postRotate(180);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_270:
+                          matrix.postRotate(270);
+                          break;
+                        default:
+                          matrix.postRotate(0);
+                      }
+                      bitmap =
+                          Bitmap.createBitmap(
+                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+                    }
+                  } catch (IOException e) {
+                    Log.e(TAG, "Bitmap rotation error:" + e);
+                  }
+                  if (bitmap != null) {
+                    faceDetection.send(bitmap);
+                  }
+                }
+              }
+            });
+    Button loadImageButton = findViewById(R.id.button_load_picture);
+    loadImageButton.setOnClickListener(
+        v -> {
+          if (inputSource != InputSource.IMAGE) {
+            stopCurrentPipeline();
+            setupStaticImageModePipeline();
+          }
+          // Reads images from gallery.
+          Intent gallery =
+              new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
+          imageGetter.launch(gallery);
+        });
+    imageView = new FaceDetectionResultImageView(this);
+  }
+
+  /** Sets up core workflow for static image mode. */
+  private void setupStaticImageModePipeline() {
+    this.inputSource = InputSource.IMAGE;
+    // Initializes a new MediaPipe Face Detection solution instance in the static image mode.
+    faceDetection =
+        new FaceDetection(
+            this,
+            FaceDetectionOptions.builder()
+                .setStaticImageMode(true)
+                .setModelSelection(0)
+                .setMinDetectionConfidence(0.5f)
+                .build());
+
+    // Connects MediaPipe Face Detection solution to the user-defined FaceDetectionResultImageView.
+    faceDetection.setResultListener(
+        faceDetectionResult -> {
+          logNoseTipKeypoint(faceDetectionResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true);
+          imageView.setFaceDetectionResult(faceDetectionResult);
+          runOnUiThread(() -> imageView.update());
+        });
+    faceDetection.setErrorListener(
+        (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+    // Updates the preview layout.
+    FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
+    frameLayout.removeAllViewsInLayout();
+    imageView.setImageDrawable(null);
+    frameLayout.addView(imageView);
+    imageView.setVisibility(View.VISIBLE);
+  }
+
+  /** Sets up the UI components for the video demo. */
+  private void setupVideoDemoUiComponents() {
+    // The Intent to access gallery and read a video file.
+    videoGetter =
+        registerForActivityResult(
+            new ActivityResultContracts.StartActivityForResult(),
+            result -> {
+              Intent resultIntent = result.getData();
+              if (resultIntent != null) {
+                if (result.getResultCode() == RESULT_OK) {
+                  glSurfaceView.post(
+                      () ->
+                          videoInput.start(
+                              this,
+                              resultIntent.getData(),
+                              faceDetection.getGlContext(),
+                              glSurfaceView.getWidth(),
+                              glSurfaceView.getHeight()));
+                }
+              }
+            });
+    Button loadVideoButton = findViewById(R.id.button_load_video);
+    loadVideoButton.setOnClickListener(
+        v -> {
+          stopCurrentPipeline();
+          setupStreamingModePipeline(InputSource.VIDEO);
+          // Reads video from gallery.
+          Intent gallery =
+              new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
+          videoGetter.launch(gallery);
+        });
+  }
+
+  /** Sets up the UI components for the live demo with camera input. */
+  private void setupLiveDemoUiComponents() {
+    Button startCameraButton = findViewById(R.id.button_start_camera);
+    startCameraButton.setOnClickListener(
+        v -> {
+          if (inputSource == InputSource.CAMERA) {
+            return;
+          }
+          stopCurrentPipeline();
+          setupStreamingModePipeline(InputSource.CAMERA);
+        });
+  }
+
+  /** Sets up core workflow for streaming mode. */
+  private void setupStreamingModePipeline(InputSource inputSource) {
+    this.inputSource = inputSource;
+    // Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
+    faceDetection =
+        new FaceDetection(
+            this,
+            FaceDetectionOptions.builder().setStaticImageMode(false).setModelSelection(0).build());
+    faceDetection.setErrorListener(
+        (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+    if (inputSource == InputSource.CAMERA) {
+      cameraInput = new CameraInput(this);
+      cameraInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
+    } else if (inputSource == InputSource.VIDEO) {
+      videoInput = new VideoInput(this);
+      videoInput.setNewFrameListener(textureFrame -> faceDetection.send(textureFrame));
+    }
+
+    // Initializes a new Gl surface view with a user-defined FaceDetectionResultGlRenderer.
+    glSurfaceView =
+        new SolutionGlSurfaceView<>(
+            this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+    glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+    glSurfaceView.setRenderInputImage(true);
+    faceDetection.setResultListener(
+        faceDetectionResult -> {
+          logNoseTipKeypoint(faceDetectionResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false);
+          glSurfaceView.setRenderData(faceDetectionResult);
+          glSurfaceView.requestRender();
+        });
+
+    // The runnable to start camera after the gl surface view is attached.
+    // For video input source, videoInput.start() will be called when the video uri is available.
+    if (inputSource == InputSource.CAMERA) {
+      glSurfaceView.post(this::startCamera);
+    }
+
+    // Updates the preview layout.
+    FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
+    imageView.setVisibility(View.GONE);
+    frameLayout.removeAllViewsInLayout();
+    frameLayout.addView(glSurfaceView);
+    glSurfaceView.setVisibility(View.VISIBLE);
+    frameLayout.requestLayout();
+  }
+
+  private void startCamera() {
+    cameraInput.start(
+        this,
+        faceDetection.getGlContext(),
+        CameraInput.CameraFacing.FRONT,
+        glSurfaceView.getWidth(),
+        glSurfaceView.getHeight());
+  }
+
+  private void stopCurrentPipeline() {
+    if (cameraInput != null) {
+      cameraInput.setNewFrameListener(null);
+      cameraInput.close();
+    }
+    if (videoInput != null) {
+      videoInput.setNewFrameListener(null);
+      videoInput.close();
+    }
+    if (glSurfaceView != null) {
+      glSurfaceView.setVisibility(View.GONE);
+    }
+    if (faceDetection != null) {
+      faceDetection.close();
+    }
+  }
+
+  private void logNoseTipKeypoint(
+      FaceDetectionResult result, int faceIndex, boolean showPixelValues) {
+    RelativeKeypoint noseTip =
+        FaceDetection.getFaceKeypoint(result, faceIndex, FaceKeypoint.NOSE_TIP);
+    // For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates.
+    if (showPixelValues) {
+      int width = result.inputBitmap().getWidth();
+      int height = result.inputBitmap().getHeight();
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
+              noseTip.getX() * width, noseTip.getY() * height));
+    } else {
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]):"
+                  + " x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+    }
+  }
+}

mediapipe/examples/android/solutions/facedetection/src/main/res

@@ -0,0 +1 @@
+../../../res

mediapipe/examples/android/solutions/facemesh/build.gradle

@@ -31,15 +31,16 @@ dependencies {
     implementation 'androidx.appcompat:appcompat:1.3.0'
     implementation 'com.google.android.material:material:1.3.0'
     implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
+    implementation 'androidx.exifinterface:exifinterface:1.3.3'
     testImplementation 'junit:junit:4.+'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
-    // MediaPipe hands solution API and solution-core.
+    // MediaPipe Face Mesh Solution components.
     implementation 'com.google.mediapipe:solution-core:latest.release'
     implementation 'com.google.mediapipe:facemesh:latest.release'
     // MediaPipe deps
-    implementation 'com.google.flogger:flogger:latest.release'
-    implementation 'com.google.flogger:flogger-system-backend:latest.release'
+    implementation 'com.google.flogger:flogger:0.6'
+    implementation 'com.google.flogger:flogger-system-backend:0.6'
     implementation 'com.google.guava:guava:27.0.1-android'
     implementation 'com.google.protobuf:protobuf-java:3.11.4'
     // CameraX core library

mediapipe/examples/android/solutions/facemesh/src/main/BUILD

@@ -38,6 +38,7 @@ android_binary(
         "//third_party:opencv",
         "@maven//:androidx_activity_activity",
         "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_exifinterface_exifinterface",
         "@maven//:androidx_fragment_fragment",
         "@maven//:com_google_guava_guava",
     ],

mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultGlRenderer.java

@@ -15,11 +15,10 @@
 package com.google.mediapipe.examples.facemesh;
 
 import android.opengl.GLES20;
-import android.opengl.Matrix;
 import com.google.common.collect.ImmutableSet;
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
-import com.google.mediapipe.solutioncore.ResultGlBoundary;
 import com.google.mediapipe.solutioncore.ResultGlRenderer;
+import com.google.mediapipe.solutions.facemesh.FaceMesh;
 import com.google.mediapipe.solutions.facemesh.FaceMeshConnections;
 import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
 import java.nio.ByteBuffer;
@@ -27,7 +26,7 @@ import java.nio.ByteOrder;
 import java.nio.FloatBuffer;
 import java.util.List;
 
-/** A custom implementation of {@link ResultGlRenderer} to render MediaPope FaceMesh results. */
+/** A custom implementation of {@link ResultGlRenderer} to render {@link FaceMeshResult}. */
 public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult> {
   private static final String TAG = "FaceMeshResultGlRenderer";
 
@@ -46,10 +45,10 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
   private static final float[] LIPS_COLOR = new float[] {0.9f, 0.9f, 0.9f, 1f};
   private static final int LIPS_THICKNESS = 8;
   private static final String VERTEX_SHADER =
-      "uniform mat4 uTransformMatrix;\n"
+      "uniform mat4 uProjectionMatrix;\n"
           + "attribute vec4 vPosition;\n"
           + "void main() {\n"
-          + "  gl_Position = uTransformMatrix * vPosition;\n"
+          + "  gl_Position = uProjectionMatrix * vPosition;\n"
           + "}";
   private static final String FRAGMENT_SHADER =
       "precision mediump float;\n"
@@ -59,9 +58,8 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
           + "}";
   private int program;
   private int positionHandle;
-  private int transformMatrixHandle;
+  private int projectionMatrixHandle;
   private int colorHandle;
-  private final float[] transformMatrix = new float[16];
 
   private int loadShader(int type, String shaderCode) {
     int shader = GLES20.glCreateShader(type);
@@ -79,28 +77,17 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
     GLES20.glAttachShader(program, fragmentShader);
     GLES20.glLinkProgram(program);
     positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
-    transformMatrixHandle = GLES20.glGetUniformLocation(program, "uTransformMatrix");
+    projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
     colorHandle = GLES20.glGetUniformLocation(program, "uColor");
   }
 
   @Override
-  public void renderResult(FaceMeshResult result, ResultGlBoundary boundary) {
+  public void renderResult(FaceMeshResult result, float[] projectionMatrix) {
     if (result == null) {
       return;
     }
     GLES20.glUseProgram(program);
-    // Sets the transform matrix to align the result rendering with the scaled output texture.
-    // Also flips the rendering vertically since OpenGL assumes the coordinate origin is at the
-    // bottom-left corner, whereas MediaPipe landmark data assumes the coordinate origin is at the
-    // top-left corner.
-    Matrix.setIdentityM(transformMatrix, 0);
-    Matrix.scaleM(
-        transformMatrix,
-        0,
-        2 / (boundary.right() - boundary.left()),
-        -2 / (boundary.top() - boundary.bottom()),
-        1.0f);
-    GLES20.glUniformMatrix4fv(transformMatrixHandle, 1, false, transformMatrix, 0);
+    GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
 
     int numFaces = result.multiFaceLandmarks().size();
     for (int i = 0; i < numFaces; ++i) {
@@ -126,7 +113,7 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
           LEFT_EYE_THICKNESS);
       drawLandmarks(
           result.multiFaceLandmarks().get(i).getLandmarkList(),
-          FaceMeshConnections.FACEMESH_LEFT_EYEBR0W,
+          FaceMeshConnections.FACEMESH_LEFT_EYEBROW,
           LEFT_EYEBROW_COLOR,
           LEFT_EYEBROW_THICKNESS);
       drawLandmarks(
@@ -139,11 +126,24 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
           FaceMeshConnections.FACEMESH_LIPS,
           LIPS_COLOR,
           LIPS_THICKNESS);
+      if (result.multiFaceLandmarks().get(i).getLandmarkCount()
+          == FaceMesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES) {
+        drawLandmarks(
+            result.multiFaceLandmarks().get(i).getLandmarkList(),
+            FaceMeshConnections.FACEMESH_RIGHT_IRIS,
+            RIGHT_EYE_COLOR,
+            RIGHT_EYE_THICKNESS);
+        drawLandmarks(
+            result.multiFaceLandmarks().get(i).getLandmarkList(),
+            FaceMeshConnections.FACEMESH_LEFT_IRIS,
+            LEFT_EYE_COLOR,
+            LEFT_EYE_THICKNESS);
+      }
     }
   }
 
   /**
-   * Calls this to delete the shader program.
+   * Deletes the shader program.
    *
    * <p>This is only necessary if one wants to release the program while keeping the context around.
    */
@@ -159,13 +159,9 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
     GLES20.glUniform4fv(colorHandle, 1, colorArray, 0);
     GLES20.glLineWidth(thickness);
     for (FaceMeshConnections.Connection c : connections) {
-      float[] vertex = new float[4];
       NormalizedLandmark start = faceLandmarkList.get(c.start());
-      vertex[0] = normalizedLandmarkValue(start.getX());
-      vertex[1] = normalizedLandmarkValue(start.getY());
       NormalizedLandmark end = faceLandmarkList.get(c.end());
-      vertex[2] = normalizedLandmarkValue(end.getX());
-      vertex[3] = normalizedLandmarkValue(end.getY());
+      float[] vertex = {start.getX(), start.getY(), end.getX(), end.getY()};
       FloatBuffer vertexBuffer =
           ByteBuffer.allocateDirect(vertex.length * 4)
               .order(ByteOrder.nativeOrder())
@@ -177,10 +173,4 @@ public class FaceMeshResultGlRenderer implements ResultGlRenderer<FaceMeshResult
       GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
     }
   }
-
-  // Normalizes the value from the landmark value range:[0, 1] to the standard OpenGL coordinate
-  // value range: [-1, 1].
-  private float normalizedLandmarkValue(float value) {
-    return value * 2 - 1;
-  }
 }

mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java

@@ -24,11 +24,12 @@ import androidx.appcompat.widget.AppCompatImageView;
 import android.util.Size;
 import com.google.common.collect.ImmutableSet;
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
+import com.google.mediapipe.solutions.facemesh.FaceMesh;
 import com.google.mediapipe.solutions.facemesh.FaceMeshConnections;
 import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
 import java.util.List;
 
-/** An ImageView implementation for displaying MediaPipe FaceMesh results. */
+/** An ImageView implementation for displaying {@link FaceMeshResult}. */
 public class FaceMeshResultImageView extends AppCompatImageView {
   private static final String TAG = "FaceMeshResultImageView";
 
@@ -103,7 +104,7 @@ public class FaceMeshResultImageView extends AppCompatImageView {
       drawLandmarksOnCanvas(
           canvas,
           result.multiFaceLandmarks().get(i).getLandmarkList(),
-          FaceMeshConnections.FACEMESH_LEFT_EYEBR0W,
+          FaceMeshConnections.FACEMESH_LEFT_EYEBROW,
           imageSize,
           LEFT_EYEBROW_COLOR,
           LEFT_EYEBROW_THICKNESS);
@@ -121,10 +122,27 @@ public class FaceMeshResultImageView extends AppCompatImageView {
           imageSize,
           LIPS_COLOR,
           LIPS_THICKNESS);
+      if (result.multiFaceLandmarks().get(i).getLandmarkCount()
+          == FaceMesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES) {
+        drawLandmarksOnCanvas(
+            canvas,
+            result.multiFaceLandmarks().get(i).getLandmarkList(),
+            FaceMeshConnections.FACEMESH_RIGHT_IRIS,
+            imageSize,
+            RIGHT_EYE_COLOR,
+            RIGHT_EYE_THICKNESS);
+        drawLandmarksOnCanvas(
+            canvas,
+            result.multiFaceLandmarks().get(i).getLandmarkList(),
+            FaceMeshConnections.FACEMESH_LEFT_IRIS,
+            imageSize,
+            LEFT_EYE_COLOR,
+            LEFT_EYE_THICKNESS);
+      }
     }
   }
 
-  /** Updates the image view with the latest facemesh result. */
+  /** Updates the image view with the latest {@link FaceMeshResult}. */
   public void update() {
     postInvalidate();
     if (latest != null) {
@@ -132,7 +150,6 @@ public class FaceMeshResultImageView extends AppCompatImageView {
     }
   }
 
-  // TODO: Better hand landmark and hand connection drawing.
   private void drawLandmarksOnCanvas(
       Canvas canvas,
       List<NormalizedLandmark> faceLandmarkList,

mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/MainActivity.java

@@ -16,6 +16,7 @@ package com.google.mediapipe.examples.facemesh;
 
 import android.content.Intent;
 import android.graphics.Bitmap;
+import android.graphics.Matrix;
 import android.os.Bundle;
 import android.provider.MediaStore;
 import androidx.appcompat.app.AppCompatActivity;
@@ -25,6 +26,8 @@ import android.widget.Button;
 import android.widget.FrameLayout;
 import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
+import androidx.exifinterface.media.ExifInterface;
+// ContentResolver dependency
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@@ -33,8 +36,9 @@ import com.google.mediapipe.solutions.facemesh.FaceMesh;
 import com.google.mediapipe.solutions.facemesh.FaceMeshOptions;
 import com.google.mediapipe.solutions.facemesh.FaceMeshResult;
 import java.io.IOException;
+import java.io.InputStream;
 
-/** Main activity of MediaPipe FaceMesh app. */
+/** Main activity of MediaPipe Face Mesh app. */
 public class MainActivity extends AppCompatActivity {
   private static final String TAG = "MainActivity";
 
@@ -57,12 +61,14 @@ public class MainActivity extends AppCompatActivity {
   private ActivityResultLauncher<Intent> videoGetter;
   // Live camera demo UI and camera components.
   private CameraInput cameraInput;
+
   private SolutionGlSurfaceView<FaceMeshResult> glSurfaceView;
 
   @Override
   protected void onCreate(Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
     setContentView(R.layout.activity_main);
+    // TODO: Add a toggle to switch between the original face mesh and attention mesh.
     setupStaticImageDemoUiComponents();
     setupVideoDemoUiComponents();
     setupLiveDemoUiComponents();
@@ -111,6 +117,35 @@ public class MainActivity extends AppCompatActivity {
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap reading error:" + e);
                   }
+                  try {
+                    InputStream imageData =
+                        this.getContentResolver().openInputStream(resultIntent.getData());
+                    int orientation =
+                        new ExifInterface(imageData)
+                            .getAttributeInt(
+                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
+                      Matrix matrix = new Matrix();
+                      switch (orientation) {
+                        case ExifInterface.ORIENTATION_ROTATE_90:
+                          matrix.postRotate(90);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_180:
+                          matrix.postRotate(180);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_270:
+                          matrix.postRotate(270);
+                          break;
+                        default:
+                          matrix.postRotate(0);
+                      }
+                      bitmap =
+                          Bitmap.createBitmap(
+                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+                    }
+                  } catch (IOException e) {
+                    Log.e(TAG, "Bitmap rotation error:" + e);
+                  }
                   if (bitmap != null) {
                     facemesh.send(bitmap);
                   }
@@ -132,26 +167,27 @@ public class MainActivity extends AppCompatActivity {
     imageView = new FaceMeshResultImageView(this);
   }
 
-  /** The core MediaPipe FaceMesh setup workflow for its static image mode. */
+  /** Sets up core workflow for static image mode. */
   private void setupStaticImageModePipeline() {
     this.inputSource = InputSource.IMAGE;
-    // Initializes a new MediaPipe FaceMesh instance in the static image mode.
+    // Initializes a new MediaPipe Face Mesh solution instance in the static image mode.
     facemesh =
         new FaceMesh(
             this,
             FaceMeshOptions.builder()
-                .setMode(FaceMeshOptions.STATIC_IMAGE_MODE)
+                .setStaticImageMode(true)
+                .setRefineLandmarks(true)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
 
-    // Connects MediaPipe FaceMesh to the user-defined FaceMeshResultImageView.
+    // Connects MediaPipe Face Mesh solution to the user-defined FaceMeshResultImageView.
     facemesh.setResultListener(
         faceMeshResult -> {
           logNoseLandmark(faceMeshResult, /*showPixelValues=*/ true);
           imageView.setFaceMeshResult(faceMeshResult);
           runOnUiThread(() -> imageView.update());
         });
-    facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+    facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
     // Updates the preview layout.
     FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
@@ -207,25 +243,24 @@ public class MainActivity extends AppCompatActivity {
         });
   }
 
-  /** The core MediaPipe FaceMesh setup workflow for its streaming mode. */
+  /** Sets up core workflow for streaming mode. */
   private void setupStreamingModePipeline(InputSource inputSource) {
     this.inputSource = inputSource;
-    // Initializes a new MediaPipe FaceMesh instance in the streaming mode.
+    // Initializes a new MediaPipe Face Mesh solution instance in the streaming mode.
     facemesh =
         new FaceMesh(
             this,
             FaceMeshOptions.builder()
-                .setMode(FaceMeshOptions.STREAMING_MODE)
+                .setStaticImageMode(false)
+                .setRefineLandmarks(true)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
-    facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+    facemesh.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
     if (inputSource == InputSource.CAMERA) {
-      // Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
       cameraInput = new CameraInput(this);
       cameraInput.setNewFrameListener(textureFrame -> facemesh.send(textureFrame));
     } else if (inputSource == InputSource.VIDEO) {
-      // Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
       videoInput = new VideoInput(this);
       videoInput.setNewFrameListener(textureFrame -> facemesh.send(textureFrame));
     }
@@ -295,13 +330,13 @@ public class MainActivity extends AppCompatActivity {
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
+              "MediaPipe Face Mesh nose coordinates (pixel values): x=%f, y=%f",
               noseLandmark.getX() * width, noseLandmark.getY() * height));
     } else {
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
     }
   }

mediapipe/examples/android/solutions/hands/build.gradle

@@ -31,15 +31,16 @@ dependencies {
     implementation 'androidx.appcompat:appcompat:1.3.0'
     implementation 'com.google.android.material:material:1.3.0'
     implementation 'androidx.constraintlayout:constraintlayout:2.0.4'
+    implementation 'androidx.exifinterface:exifinterface:1.3.3'
     testImplementation 'junit:junit:4.+'
     androidTestImplementation 'androidx.test.ext:junit:1.1.2'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'
-    // MediaPipe hands solution API and solution-core.
+    // MediaPipe Hands Solution components.
     implementation 'com.google.mediapipe:solution-core:latest.release'
     implementation 'com.google.mediapipe:hands:latest.release'
     // MediaPipe deps
-    implementation 'com.google.flogger:flogger:latest.release'
-    implementation 'com.google.flogger:flogger-system-backend:latest.release'
+    implementation 'com.google.flogger:flogger:0.6'
+    implementation 'com.google.flogger:flogger-system-backend:0.6'
     implementation 'com.google.guava:guava:27.0.1-android'
     implementation 'com.google.protobuf:protobuf-java:3.11.4'
     // CameraX core library

mediapipe/examples/android/solutions/hands/src/main/BUILD

@@ -38,6 +38,7 @@ android_binary(
         "//third_party:opencv",
         "@maven//:androidx_activity_activity",
         "@maven//:androidx_concurrent_concurrent_futures",
+        "@maven//:androidx_exifinterface_exifinterface",
         "@maven//:androidx_fragment_fragment",
         "@maven//:com_google_guava_guava",
     ],

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java

@@ -15,9 +15,7 @@
 package com.google.mediapipe.examples.hands;
 
 import android.opengl.GLES20;
-import android.opengl.Matrix;
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
-import com.google.mediapipe.solutioncore.ResultGlBoundary;
 import com.google.mediapipe.solutioncore.ResultGlRenderer;
 import com.google.mediapipe.solutions.hands.Hands;
 import com.google.mediapipe.solutions.hands.HandsResult;
@@ -26,16 +24,16 @@ import java.nio.ByteOrder;
 import java.nio.FloatBuffer;
 import java.util.List;
 
-/** A custom implementation of {@link ResultGlRenderer} to render MediaPope Hands results. */
+/** A custom implementation of {@link ResultGlRenderer} to render {@link HandsResult}. */
 public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
   private static final String TAG = "HandsResultGlRenderer";
 
   private static final float CONNECTION_THICKNESS = 20.0f;
   private static final String VERTEX_SHADER =
-      "uniform mat4 uTransformMatrix;\n"
+      "uniform mat4 uProjectionMatrix;\n"
           + "attribute vec4 vPosition;\n"
           + "void main() {\n"
-          + "  gl_Position = uTransformMatrix * vPosition;\n"
+          + "  gl_Position = uProjectionMatrix * vPosition;\n"
           + "}";
   private static final String FRAGMENT_SHADER =
       "precision mediump float;\n"
@@ -44,8 +42,7 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
           + "}";
   private int program;
   private int positionHandle;
-  private int transformMatrixHandle;
-  private final float[] transformMatrix = new float[16];
+  private int projectionMatrixHandle;
 
   private int loadShader(int type, String shaderCode) {
     int shader = GLES20.glCreateShader(type);
@@ -63,27 +60,16 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
     GLES20.glAttachShader(program, fragmentShader);
     GLES20.glLinkProgram(program);
     positionHandle = GLES20.glGetAttribLocation(program, "vPosition");
-    transformMatrixHandle = GLES20.glGetUniformLocation(program, "uTransformMatrix");
+    projectionMatrixHandle = GLES20.glGetUniformLocation(program, "uProjectionMatrix");
   }
 
   @Override
-  public void renderResult(HandsResult result, ResultGlBoundary boundary) {
+  public void renderResult(HandsResult result, float[] projectionMatrix) {
     if (result == null) {
       return;
     }
     GLES20.glUseProgram(program);
-    // Sets the transform matrix to align the result rendering with the scaled output texture.
-    // Also flips the rendering vertically since OpenGL assumes the coordinate origin is at the
-    // bottom-left corner, whereas MediaPipe landmark data assumes the coordinate origin is at the
-    // top-left corner.
-    Matrix.setIdentityM(transformMatrix, 0);
-    Matrix.scaleM(
-        transformMatrix,
-        0,
-        2 / (boundary.right() - boundary.left()),
-        -2 / (boundary.top() - boundary.bottom()),
-        1.0f);
-    GLES20.glUniformMatrix4fv(transformMatrixHandle, 1, false, transformMatrix, 0);
+    GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
     GLES20.glLineWidth(CONNECTION_THICKNESS);
 
     int numHands = result.multiHandLandmarks().size();
@@ -93,7 +79,7 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
   }
 
   /**
-   * Calls this to delete the shader program.
+   * Deletes the shader program.
    *
    * <p>This is only necessary if one wants to release the program while keeping the context around.
    */
@@ -101,16 +87,11 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
     GLES20.glDeleteProgram(program);
   }
 
-  // TODO: Better hand landmark and hand connection drawing.
   private void drawLandmarks(List<NormalizedLandmark> handLandmarkList) {
     for (Hands.Connection c : Hands.HAND_CONNECTIONS) {
-      float[] vertex = new float[4];
       NormalizedLandmark start = handLandmarkList.get(c.start());
-      vertex[0] = normalizedLandmarkValue(start.getX());
-      vertex[1] = normalizedLandmarkValue(start.getY());
       NormalizedLandmark end = handLandmarkList.get(c.end());
-      vertex[2] = normalizedLandmarkValue(end.getX());
-      vertex[3] = normalizedLandmarkValue(end.getY());
+      float[] vertex = {start.getX(), start.getY(), end.getX(), end.getY()};
       FloatBuffer vertexBuffer =
           ByteBuffer.allocateDirect(vertex.length * 4)
               .order(ByteOrder.nativeOrder())
@@ -122,10 +103,4 @@ public class HandsResultGlRenderer implements ResultGlRenderer<HandsResult> {
       GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2);
     }
   }
-
-  // Normalizes the value from the landmark value range:[0, 1] to the standard OpenGL coordinate
-  // value range: [-1, 1].
-  private float normalizedLandmarkValue(float value) {
-    return value * 2 - 1;
-  }
 }

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java

@@ -27,7 +27,7 @@ import com.google.mediapipe.solutions.hands.Hands;
 import com.google.mediapipe.solutions.hands.HandsResult;
 import java.util.List;
 
-/** An ImageView implementation for displaying MediaPipe Hands results. */
+/** An ImageView implementation for displaying {@link HandsResult}. */
 public class HandsResultImageView extends AppCompatImageView {
   private static final String TAG = "HandsResultImageView";
 
@@ -66,7 +66,7 @@ public class HandsResultImageView extends AppCompatImageView {
     }
   }
 
-  /** Updates the image view with the latest hands result. */
+  /** Updates the image view with the latest {@link HandsResult}. */
   public void update() {
     postInvalidate();
     if (latest != null) {
@@ -74,7 +74,6 @@ public class HandsResultImageView extends AppCompatImageView {
     }
   }
 
-  // TODO: Better hand landmark and hand connection drawing.
   private void drawLandmarksOnCanvas(
       List<NormalizedLandmark> handLandmarkList, Canvas canvas, int width, int height) {
     // Draw connections.

mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java

@@ -16,6 +16,7 @@ package com.google.mediapipe.examples.hands;
 
 import android.content.Intent;
 import android.graphics.Bitmap;
+import android.graphics.Matrix;
 import android.os.Bundle;
 import android.provider.MediaStore;
 import androidx.appcompat.app.AppCompatActivity;
@@ -25,6 +26,8 @@ import android.widget.Button;
 import android.widget.FrameLayout;
 import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
+import androidx.exifinterface.media.ExifInterface;
+// ContentResolver dependency
 import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@@ -34,6 +37,7 @@ import com.google.mediapipe.solutions.hands.Hands;
 import com.google.mediapipe.solutions.hands.HandsOptions;
 import com.google.mediapipe.solutions.hands.HandsResult;
 import java.io.IOException;
+import java.io.InputStream;
 
 /** Main activity of MediaPipe Hands app. */
 public class MainActivity extends AppCompatActivity {
@@ -59,6 +63,7 @@ public class MainActivity extends AppCompatActivity {
   private ActivityResultLauncher<Intent> videoGetter;
   // Live camera demo UI and camera components.
   private CameraInput cameraInput;
+
   private SolutionGlSurfaceView<HandsResult> glSurfaceView;
 
   @Override
@@ -113,6 +118,35 @@ public class MainActivity extends AppCompatActivity {
                   } catch (IOException e) {
                     Log.e(TAG, "Bitmap reading error:" + e);
                   }
+                  try {
+                    InputStream imageData =
+                        this.getContentResolver().openInputStream(resultIntent.getData());
+                    int orientation =
+                        new ExifInterface(imageData)
+                            .getAttributeInt(
+                                ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL);
+                    if (orientation != ExifInterface.ORIENTATION_NORMAL) {
+                      Matrix matrix = new Matrix();
+                      switch (orientation) {
+                        case ExifInterface.ORIENTATION_ROTATE_90:
+                          matrix.postRotate(90);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_180:
+                          matrix.postRotate(180);
+                          break;
+                        case ExifInterface.ORIENTATION_ROTATE_270:
+                          matrix.postRotate(270);
+                          break;
+                        default:
+                          matrix.postRotate(0);
+                      }
+                      bitmap =
+                          Bitmap.createBitmap(
+                              bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+                    }
+                  } catch (IOException e) {
+                    Log.e(TAG, "Bitmap rotation error:" + e);
+                  }
                   if (bitmap != null) {
                     hands.send(bitmap);
                   }
@@ -134,20 +168,20 @@ public class MainActivity extends AppCompatActivity {
     imageView = new HandsResultImageView(this);
   }
 
-  /** The core MediaPipe Hands setup workflow for its static image mode. */
+  /** Sets up core workflow for static image mode. */
   private void setupStaticImageModePipeline() {
     this.inputSource = InputSource.IMAGE;
-    // Initializes a new MediaPipe Hands instance in the static image mode.
+    // Initializes a new MediaPipe Hands solution instance in the static image mode.
     hands =
         new Hands(
             this,
             HandsOptions.builder()
-                .setMode(HandsOptions.STATIC_IMAGE_MODE)
+                .setStaticImageMode(true)
                 .setMaxNumHands(1)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
 
-    // Connects MediaPipe Hands to the user-defined HandsResultImageView.
+    // Connects MediaPipe Hands solution to the user-defined HandsResultImageView.
     hands.setResultListener(
         handsResult -> {
           logWristLandmark(handsResult, /*showPixelValues=*/ true);
@@ -210,26 +244,24 @@ public class MainActivity extends AppCompatActivity {
         });
   }
 
-  /** The core MediaPipe Hands setup workflow for its streaming mode. */
+  /** Sets up core workflow for streaming mode. */
   private void setupStreamingModePipeline(InputSource inputSource) {
     this.inputSource = inputSource;
-    // Initializes a new MediaPipe Hands instance in the streaming mode.
+    // Initializes a new MediaPipe Hands solution instance in the streaming mode.
     hands =
         new Hands(
             this,
             HandsOptions.builder()
-                .setMode(HandsOptions.STREAMING_MODE)
+                .setStaticImageMode(false)
                 .setMaxNumHands(1)
                 .setRunOnGpu(RUN_ON_GPU)
                 .build());
     hands.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
     if (inputSource == InputSource.CAMERA) {
-      // Initializes a new CameraInput instance and connects it to MediaPipe Hands.
       cameraInput = new CameraInput(this);
       cameraInput.setNewFrameListener(textureFrame -> hands.send(textureFrame));
     } else if (inputSource == InputSource.VIDEO) {
-      // Initializes a new VideoInput instance and connects it to MediaPipe Hands.
       videoInput = new VideoInput(this);
       videoInput.setNewFrameListener(textureFrame -> hands.send(textureFrame));
     }

mediapipe/examples/android/solutions/settings.gradle

@@ -1,3 +1,4 @@
 rootProject.name = "mediapipe-solutions-examples"
-include ':hands'
+include ':facedetection'
 include ':facemesh'
+include ':hands'

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD

@@ -37,7 +37,7 @@ android_binary(
     srcs = glob(["*.java"]),
     assets = [
         "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
-        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
         "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
     ],
     assets_dir = "",

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -63,7 +63,7 @@ objc_library(
     data = [
         "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
         "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
-        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
     ],
     deps = [
         "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary",

mediapipe/framework/api2/node.h

@@ -223,24 +223,23 @@ class SubgraphImpl : public Subgraph, public Intf {
 
 // This macro is used to register a calculator that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                    \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken> \
-      REGISTRY_STATIC_VAR(calculator_registration, __LINE__)(  \
-          mediapipe::CalculatorBaseRegistry::Register(         \
-              Impl::kCalculatorName,                           \
-              absl::make_unique<                               \
-                  mediapipe::internal::CalculatorBaseFactoryFor<Impl>>))
+#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                  \
+  static mediapipe::NoDestructor<mediapipe::RegistrationToken>               \
+  REGISTRY_STATIC_VAR(calculator_registration,                               \
+                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register( \
+      Impl::kCalculatorName,                                                 \
+      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>))
 
 // This macro is used to register a non-split-contract calculator. Deprecated.
 #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
 
 // This macro is used to define a subgraph that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)                            \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>             \
-      REGISTRY_STATIC_VAR(subgraph_registration,                           \
-                          __LINE__)(mediapipe::SubgraphRegistry::Register( \
-          Impl::kCalculatorName, absl::make_unique<Impl>))
+#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)                        \
+  static mediapipe::NoDestructor<mediapipe::RegistrationToken>         \
+  REGISTRY_STATIC_VAR(subgraph_registration,                           \
+                      __LINE__)(mediapipe::SubgraphRegistry::Register( \
+      Impl::kCalculatorName, absl::make_unique<Impl>))
 
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/api2/port.h

@@ -454,12 +454,12 @@ class OutputShardAccessBase {
     if (output_) output_->SetNextTimestampBound(timestamp);
   }
 
-  bool IsClosed() { return output_ ? output_->IsClosed() : true; }
+  bool IsClosed() const { return output_ ? output_->IsClosed() : true; }
   void Close() {
     if (output_) output_->Close();
   }
 
-  bool IsConnected() { return output_ != nullptr; }
+  bool IsConnected() const { return output_ != nullptr; }
 
  protected:
   const CalculatorContext& context_;
@@ -559,7 +559,7 @@ class InputShardAccess : public Packet<T> {
   PacketBase packet() const&& { return *this; }
 
   bool IsDone() const { return stream_->IsDone(); }
-  bool IsConnected() { return stream_ != nullptr; }
+  bool IsConnected() const { return stream_ != nullptr; }
 
   PacketBase Header() const { return FromOldPacket(stream_->Header()); }
 
@@ -619,7 +619,7 @@ class InputSidePacketAccess : public Packet<T> {
   const PacketBase& packet() const& { return *this; }
   PacketBase packet() const&& { return *this; }
 
-  bool IsConnected() { return connected_; }
+  bool IsConnected() const { return connected_; }
 
  private:
   InputSidePacketAccess(const mediapipe::Packet* packet)
@@ -639,8 +639,8 @@ class InputShardOrSideAccess : public Packet<T> {
   PacketBase packet() const&& { return *this; }
 
   bool IsDone() const { return stream_->IsDone(); }
-  bool IsConnected() { return connected_; }
-  bool IsStream() { return stream_ != nullptr; }
+  bool IsConnected() const { return connected_; }
+  bool IsStream() const { return stream_ != nullptr; }
 
   PacketBase Header() const { return FromOldPacket(stream_->Header()); }
 
@@ -662,7 +662,7 @@ class InputShardOrSideAccess : public Packet<T> {
 
 class PacketTypeAccess {
  public:
-  bool IsConnected() { return packet_type_ != nullptr; }
+  bool IsConnected() const { return packet_type_ != nullptr; }
 
  protected:
   PacketTypeAccess(PacketType* pt) : packet_type_(pt) {}
@@ -675,7 +675,7 @@ class PacketTypeAccess {
 
 class PacketTypeAccessFallback : public PacketTypeAccess {
  public:
-  bool IsStream() { return is_stream_; }
+  bool IsStream() const { return is_stream_; }
 
  private:
   PacketTypeAccessFallback(PacketType* pt, bool is_stream)

mediapipe/framework/calculator.proto

@@ -321,6 +321,8 @@ message CalculatorGraphConfig {
     // The maximum number of invocations that can be executed in parallel.
     // If not specified, the limit is one invocation.
     int32 max_in_flight = 16;
+    // Defines an option value for this Node from graph options or packets.
+    repeated string option_value = 17;
     // DEPRECATED: For backwards compatibility we allow users to
     // specify the old name for "input_side_packet" in proto configs.
     // These are automatically converted to input_side_packets during

mediapipe/framework/calculator_graph.cc

@@ -465,7 +465,7 @@ absl::Status CalculatorGraph::ObserveOutputStream(
 }
 
 absl::StatusOr<OutputStreamPoller> CalculatorGraph::AddOutputStreamPoller(
-    const std::string& stream_name) {
+    const std::string& stream_name, bool observe_timestamp_bounds) {
   RET_CHECK(initialized_).SetNoLogging()
       << "CalculatorGraph is not initialized.";
   int output_stream_index = validated_graph_->OutputStreamIndex(stream_name);
@@ -479,7 +479,7 @@ absl::StatusOr<OutputStreamPoller> CalculatorGraph::AddOutputStreamPoller(
       stream_name, &any_packet_type_,
       std::bind(&CalculatorGraph::UpdateThrottledNodes, this,
                 std::placeholders::_1, std::placeholders::_2),
-      &output_stream_managers_[output_stream_index]));
+      &output_stream_managers_[output_stream_index], observe_timestamp_bounds));
   OutputStreamPoller poller(internal_poller);
   graph_output_streams_.push_back(std::move(internal_poller));
   return std::move(poller);

mediapipe/framework/calculator_graph.h

@@ -164,7 +164,8 @@ class CalculatorGraph {
   // polling API for accessing a stream's output. Should only be called before
   // Run() or StartRun(). For asynchronous output, use ObserveOutputStream. See
   // also the helpers in tool/sink.h.
-  StatusOrPoller AddOutputStreamPoller(const std::string& stream_name);
+  StatusOrPoller AddOutputStreamPoller(const std::string& stream_name,
+                                       bool observe_timestamp_bounds = false);
 
   // Gets output side packet by name after the graph is done. However, base
   // packets (generated by PacketGenerators) can be retrieved before

mediapipe/framework/calculator_graph_test.cc

@@ -4348,5 +4348,349 @@ TEST(CalculatorGraph, GraphInputStreamWithTag) {
   ASSERT_EQ(5, packet_dump.size());
 }
 
+TEST(CalculatorGraph, GraphInputStreamBeforeStartRun) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: "VIDEO_METADATA:video_metadata"
+        input_stream: "max_count"
+        node {
+          calculator: "PassThroughCalculator"
+          input_stream: "FIRST_INPUT:video_metadata"
+          input_stream: "max_count"
+          output_stream: "FIRST_INPUT:output_0"
+          output_stream: "output_1"
+        }
+      )pb");
+  std::vector<Packet> packet_dump;
+  tool::AddVectorSink("output_0", &config, &packet_dump);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  ASSERT_EQ(graph
+                .AddPacketToInputStream("video_metadata",
+                                        MakePacket<int>(0).At(Timestamp(0)))
+                .code(),
+            absl::StatusCode::kFailedPrecondition);
+}
+
+// Returns the first packet of the input stream.
+class FirstPacketFilterCalculator : public CalculatorBase {
+ public:
+  FirstPacketFilterCalculator() {}
+  ~FirstPacketFilterCalculator() override {}
+
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (!seen_first_packet_) {
+      cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
+      cc->Outputs().Index(0).Close();
+      seen_first_packet_ = true;
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  bool seen_first_packet_ = false;
+};
+REGISTER_CALCULATOR(FirstPacketFilterCalculator);
+constexpr int kDefaultMaxCount = 1000;
+
+TEST(CalculatorGraph, TestPollPacket) {
+  CalculatorGraphConfig config;
+  CalculatorGraphConfig::Node* node = config.add_node();
+  node->set_calculator("CountingSourceCalculator");
+  node->add_output_stream("output");
+  node->add_input_side_packet("MAX_COUNT:max_count");
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto status_or_poller = graph.AddOutputStreamPoller("output");
+  ASSERT_TRUE(status_or_poller.ok());
+  OutputStreamPoller poller = std::move(status_or_poller.value());
+  MP_ASSERT_OK(
+      graph.StartRun({{"max_count", MakePacket<int>(kDefaultMaxCount)}}));
+  Packet packet;
+  int num_packets = 0;
+  while (poller.Next(&packet)) {
+    EXPECT_EQ(num_packets, packet.Get<int>());
+    ++num_packets;
+  }
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_FALSE(poller.Next(&packet));
+  EXPECT_EQ(kDefaultMaxCount, num_packets);
+}
+
+TEST(CalculatorGraph, TestOutputStreamPollerDesiredQueueSize) {
+  CalculatorGraphConfig config;
+  CalculatorGraphConfig::Node* node = config.add_node();
+  node->set_calculator("CountingSourceCalculator");
+  node->add_output_stream("output");
+  node->add_input_side_packet("MAX_COUNT:max_count");
+
+  for (int queue_size = 1; queue_size < 10; ++queue_size) {
+    CalculatorGraph graph;
+    MP_ASSERT_OK(graph.Initialize(config));
+    auto status_or_poller = graph.AddOutputStreamPoller("output");
+    ASSERT_TRUE(status_or_poller.ok());
+    OutputStreamPoller poller = std::move(status_or_poller.value());
+    poller.SetMaxQueueSize(queue_size);
+    MP_ASSERT_OK(
+        graph.StartRun({{"max_count", MakePacket<int>(kDefaultMaxCount)}}));
+    Packet packet;
+    int num_packets = 0;
+    while (poller.Next(&packet)) {
+      EXPECT_EQ(num_packets, packet.Get<int>());
+      ++num_packets;
+    }
+    MP_ASSERT_OK(graph.CloseAllPacketSources());
+    MP_ASSERT_OK(graph.WaitUntilDone());
+    EXPECT_FALSE(poller.Next(&packet));
+    EXPECT_EQ(kDefaultMaxCount, num_packets);
+  }
+}
+
+TEST(CalculatorGraph, TestPollPacketsFromMultipleStreams) {
+  CalculatorGraphConfig config;
+  CalculatorGraphConfig::Node* node1 = config.add_node();
+  node1->set_calculator("CountingSourceCalculator");
+  node1->add_output_stream("stream1");
+  node1->add_input_side_packet("MAX_COUNT:max_count");
+  CalculatorGraphConfig::Node* node2 = config.add_node();
+  node2->set_calculator("PassThroughCalculator");
+  node2->add_input_stream("stream1");
+  node2->add_output_stream("stream2");
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  auto status_or_poller1 = graph.AddOutputStreamPoller("stream1");
+  ASSERT_TRUE(status_or_poller1.ok());
+  OutputStreamPoller poller1 = std::move(status_or_poller1.value());
+  auto status_or_poller2 = graph.AddOutputStreamPoller("stream2");
+  ASSERT_TRUE(status_or_poller2.ok());
+  OutputStreamPoller poller2 = std::move(status_or_poller2.value());
+  MP_ASSERT_OK(
+      graph.StartRun({{"max_count", MakePacket<int>(kDefaultMaxCount)}}));
+  Packet packet1;
+  Packet packet2;
+  int num_packets1 = 0;
+  int num_packets2 = 0;
+  int running_pollers = 2;
+  while (running_pollers > 0) {
+    if (poller1.Next(&packet1)) {
+      EXPECT_EQ(num_packets1++, packet1.Get<int>());
+    } else {
+      --running_pollers;
+    }
+    if (poller2.Next(&packet2)) {
+      EXPECT_EQ(num_packets2++, packet2.Get<int>());
+    } else {
+      --running_pollers;
+    }
+  }
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_FALSE(poller1.Next(&packet1));
+  EXPECT_FALSE(poller2.Next(&packet2));
+  EXPECT_EQ(kDefaultMaxCount, num_packets1);
+  EXPECT_EQ(kDefaultMaxCount, num_packets2);
+}
+
+class TimestampBoundTestCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->Outputs().Index(0).Set<int>();
+    return absl::OkStatus();
+  }
+  absl::Status Open(CalculatorContext* cc) final { return absl::OkStatus(); }
+  absl::Status Process(CalculatorContext* cc) final {
+    if (count_ % 50 == 1) {
+      // Outputs packets at t10 and t60.
+      cc->Outputs().Index(0).AddPacket(
+          MakePacket<int>(count_).At(Timestamp(count_)));
+    } else if (count_ % 15 == 7) {
+      cc->Outputs().Index(0).SetNextTimestampBound(Timestamp(count_));
+    }
+    absl::SleepFor(absl::Milliseconds(3));
+    ++count_;
+    if (count_ == 110) {
+      return tool::StatusStop();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  int count_ = 0;
+};
+REGISTER_CALCULATOR(TimestampBoundTestCalculator);
+
+TEST(CalculatorGraph, TestPollPacketsWithTimestampNotification) {
+  std::string config_str = R"(
+            node {
+              calculator: "TimestampBoundTestCalculator"
+              output_stream: "foo"
+            }
+          )";
+  CalculatorGraphConfig graph_config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(config_str);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  auto status_or_poller =
+      graph.AddOutputStreamPoller("foo", /*observe_timestamp_bounds=*/true);
+  ASSERT_TRUE(status_or_poller.ok());
+  OutputStreamPoller poller = std::move(status_or_poller.value());
+  Packet packet;
+  std::vector<int> timestamps;
+  std::vector<int> values;
+  MP_ASSERT_OK(graph.StartRun({}));
+  while (poller.Next(&packet)) {
+    if (packet.IsEmpty()) {
+      timestamps.push_back(packet.Timestamp().Value());
+    } else {
+      values.push_back(packet.Get<int>());
+    }
+  }
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  ASSERT_FALSE(poller.Next(&packet));
+  ASSERT_FALSE(timestamps.empty());
+  int prev_t = 0;
+  for (auto t : timestamps) {
+    EXPECT_TRUE(t > prev_t && t < 110);
+    prev_t = t;
+  }
+  ASSERT_EQ(3, values.size());
+  EXPECT_EQ(1, values[0]);
+  EXPECT_EQ(51, values[1]);
+  EXPECT_EQ(101, values[2]);
+}
+
+// Ensure that when a custom input stream handler is used to handle packets from
+// input streams, an error message is outputted with the appropriate link to
+// resolve the issue when the calculator doesn't handle inputs in monotonically
+// increasing order of timestamps.
+TEST(CalculatorGraph, SimpleMuxCalculatorWithCustomInputStreamHandler) {
+  CalculatorGraph graph;
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input0'
+        input_stream: 'input1'
+        node {
+          calculator: 'SimpleMuxCalculator'
+          input_stream: 'input0'
+          input_stream: 'input1'
+          input_stream_handler {
+            input_stream_handler: "ImmediateInputStreamHandler"
+          }
+          output_stream: 'output'
+        }
+      )pb");
+  std::vector<Packet> packet_dump;
+  tool::AddVectorSink("output", &config, &packet_dump);
+
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Send packets to input stream "input0" at timestamps 0 and 1 consecutively.
+  Timestamp input0_timestamp = Timestamp(0);
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "input0", MakePacket<int>(1).At(input0_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  ASSERT_EQ(1, packet_dump.size());
+  EXPECT_EQ(1, packet_dump[0].Get<int>());
+
+  ++input0_timestamp;
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "input0", MakePacket<int>(3).At(input0_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  ASSERT_EQ(2, packet_dump.size());
+  EXPECT_EQ(3, packet_dump[1].Get<int>());
+
+  // Send a packet to input stream "input1" at timestamp 0 after sending two
+  // packets at timestamps 0 and 1 to input stream "input0". This will result
+  // in a mismatch in timestamps as the SimpleMuxCalculator doesn't handle
+  // inputs from all streams in monotonically increasing order of timestamps.
+  Timestamp input1_timestamp = Timestamp(0);
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "input1", MakePacket<int>(2).At(input1_timestamp)));
+  absl::Status run_status = graph.WaitUntilIdle();
+  EXPECT_THAT(
+      run_status.ToString(),
+      testing::AllOf(
+          // The core problem.
+          testing::HasSubstr("timestamp mismatch on a calculator"),
+          testing::HasSubstr(
+              "timestamps that are not strictly monotonically increasing"),
+          // Link to the possible solution.
+          testing::HasSubstr("ImmediateInputStreamHandler class comment")));
+}
+
+void DoTestMultipleGraphRuns(absl::string_view input_stream_handler,
+                             bool select_packet) {
+  std::string graph_proto = absl::StrFormat(R"(
+    input_stream: 'input'
+    input_stream: 'select'
+    node {
+      calculator: 'PassThroughCalculator'
+      input_stream: 'input'
+      input_stream: 'select'
+      input_stream_handler {
+        input_stream_handler: "%s"
+      }
+      output_stream: 'output'
+      output_stream: 'select_out'
+    }
+  )",
+                                            input_stream_handler.data());
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
+  std::vector<Packet> packet_dump;
+  tool::AddVectorSink("output", &config, &packet_dump);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  struct Run {
+    Timestamp timestamp;
+    int value;
+  };
+  std::vector<Run> runs = {{.timestamp = Timestamp(2000), .value = 2},
+                           {.timestamp = Timestamp(1000), .value = 1}};
+  for (const Run& run : runs) {
+    MP_ASSERT_OK(graph.StartRun({}));
+
+    if (select_packet) {
+      MP_EXPECT_OK(graph.AddPacketToInputStream(
+          "select", MakePacket<int>(0).At(run.timestamp)));
+    }
+    MP_EXPECT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(run.value).At(run.timestamp)));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+    ASSERT_EQ(1, packet_dump.size());
+    EXPECT_EQ(run.value, packet_dump[0].Get<int>());
+    EXPECT_EQ(run.timestamp, packet_dump[0].Timestamp());
+
+    MP_ASSERT_OK(graph.CloseAllPacketSources());
+    MP_ASSERT_OK(graph.WaitUntilDone());
+
+    packet_dump.clear();
+  }
+}
+
+TEST(CalculatorGraph, MultipleRunsWithDifferentInputStreamHandlers) {
+  DoTestMultipleGraphRuns("BarrierInputStreamHandler", true);
+  DoTestMultipleGraphRuns("DefaultInputStreamHandler", true);
+  DoTestMultipleGraphRuns("EarlyCloseInputStreamHandler", true);
+  DoTestMultipleGraphRuns("FixedSizeInputStreamHandler", true);
+  DoTestMultipleGraphRuns("ImmediateInputStreamHandler", false);
+  DoTestMultipleGraphRuns("MuxInputStreamHandler", true);
+  DoTestMultipleGraphRuns("SyncSetInputStreamHandler", true);
+  DoTestMultipleGraphRuns("TimestampAlignInputStreamHandler", true);
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/deps/file_path.cc

@@ -45,7 +45,7 @@ std::string JoinPathImpl(bool honor_abs,
     // This size calculation is worst-case: it assumes one extra "/" for every
     // path other than the first.
     size_t total_size = paths.size() - 1;
-    for (const absl::string_view path : paths) total_size += path.size();
+    for (const absl::string_view& path : paths) total_size += path.size();
     result.resize(total_size);
 
     auto begin = result.begin();

mediapipe/framework/formats/BUILD

@@ -81,6 +81,12 @@ mediapipe_proto_library(
     deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
 )
 
+mediapipe_proto_library(
+    name = "affine_transform_data_proto",
+    srcs = ["affine_transform_data.proto"],
+    visibility = ["//visibility:public"],
+)
+
 mediapipe_proto_library(
     name = "time_series_header_proto",
     srcs = ["time_series_header.proto"],
@@ -119,6 +125,31 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "affine_transform",
+    srcs = ["affine_transform.cc"],
+    hdrs = ["affine_transform.h"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:port",
+        "//mediapipe/framework:type_map",
+        "//mediapipe/framework/formats:affine_transform_data_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:point",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/framework/tool:status_util",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "@com_google_protobuf//:protobuf",
+    ],
+)
+
 cc_library(
     name = "image_frame",
     srcs = ["image_frame.cc"],

mediapipe/framework/formats/affine_transform.cc

@@ -0,0 +1,228 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/affine_transform.h"
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/point2.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "mediapipe/framework/type_map.h"
+
+namespace mediapipe {
+using ::mediapipe::AffineTransformData;
+
+AffineTransform::AffineTransform() { SetScale(Point2_f(1, 1)); }
+
+AffineTransform::AffineTransform(
+    const AffineTransformData& affine_transform_data)
+    : affine_transform_data_(affine_transform_data), is_dirty_(true) {
+  // make sure scale is set to default (1, 1) when none provided
+  if (!affine_transform_data_.has_scale()) {
+    SetScale(Point2_f(1, 1));
+  }
+}
+
+AffineTransform AffineTransform::Create(const Point2_f& translation,
+                                        const Point2_f& scale, float rotation,
+                                        const Point2_f& shear) {
+  AffineTransformData affine_transform_data;
+
+  auto* t = affine_transform_data.mutable_translation();
+  t->set_x(translation.x());
+  t->set_y(translation.y());
+
+  auto* s = affine_transform_data.mutable_scale();
+  s->set_x(scale.x());
+  s->set_y(scale.y());
+
+  s = affine_transform_data.mutable_shear();
+  s->set_x(shear.x());
+  s->set_y(shear.y());
+
+  affine_transform_data.set_rotation(rotation);
+
+  return AffineTransform(affine_transform_data);
+}
+
+// Accessor for the composition matrix
+std::vector<float> AffineTransform::GetCompositionMatrix() {
+  float r = affine_transform_data_.rotation();
+  const auto t = affine_transform_data_.translation();
+  const auto sc = affine_transform_data_.scale();
+  const auto sh = affine_transform_data_.shear();
+
+  if (is_dirty_) {
+    // Composition matrix M = T*R*Sh*Sc
+    // Column based to match GL matrix store order
+    float cos_r = std::cos(r);
+    float sin_r = std::sin(r);
+    matrix_[0] = (cos_r + sin_r * -sh.y()) * sc.x();
+    matrix_[1] = (-sin_r + cos_r * -sh.y()) * sc.x();
+    matrix_[2] = 0;
+    matrix_[3] = (cos_r * -sh.x() + sin_r) * sc.y();
+    matrix_[4] = (-sin_r * -sh.x() + cos_r) * sc.y();
+    matrix_[5] = 0;
+    matrix_[6] = t.x();
+    matrix_[7] = -t.y();
+    matrix_[8] = 1;
+    is_dirty_ = false;
+  }
+
+  return matrix_;
+}
+
+Point2_f AffineTransform::GetScale() const {
+  return Point2_f(affine_transform_data_.scale().x(),
+                  affine_transform_data_.scale().y());
+}
+
+Point2_f AffineTransform::GetTranslation() const {
+  return Point2_f(affine_transform_data_.translation().x(),
+                  affine_transform_data_.translation().y());
+}
+
+Point2_f AffineTransform::GetShear() const {
+  return Point2_f(affine_transform_data_.shear().x(),
+                  affine_transform_data_.shear().y());
+}
+
+float AffineTransform::GetRotation() const {
+  return affine_transform_data_.rotation();
+}
+
+void AffineTransform::SetScale(const Point2_f& scale) {
+  auto* s = affine_transform_data_.mutable_scale();
+  s->set_x(scale.x());
+  s->set_y(scale.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::SetTranslation(const Point2_f& translation) {
+  auto* t = affine_transform_data_.mutable_translation();
+  t->set_x(translation.x());
+  t->set_y(translation.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::SetShear(const Point2_f& shear) {
+  auto* s = affine_transform_data_.mutable_shear();
+  s->set_x(shear.x());
+  s->set_y(shear.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::SetRotation(float rotationInRadians) {
+  affine_transform_data_.set_rotation(rotationInRadians);
+  is_dirty_ = true;
+}
+
+void AffineTransform::AddScale(const Point2_f& scale) {
+  auto* s = affine_transform_data_.mutable_scale();
+  s->set_x(s->x() + scale.x());
+  s->set_y(s->y() + scale.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::AddTranslation(const Point2_f& translation) {
+  auto* t = affine_transform_data_.mutable_translation();
+  t->set_x(t->x() + translation.x());
+  t->set_y(t->y() + translation.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::AddShear(const Point2_f& shear) {
+  auto* s = affine_transform_data_.mutable_shear();
+  s->set_x(s->x() + shear.x());
+  s->set_y(s->y() + shear.y());
+  is_dirty_ = true;
+}
+
+void AffineTransform::AddRotation(float rotationInRadians) {
+  affine_transform_data_.set_rotation(affine_transform_data_.rotation() +
+                                      rotationInRadians);
+  is_dirty_ = true;
+}
+
+void AffineTransform::SetFromProto(const AffineTransformData& proto) {
+  affine_transform_data_ = proto;
+}
+
+void AffineTransform::ConvertToProto(AffineTransformData* proto) const {
+  *proto = affine_transform_data_;
+}
+
+AffineTransformData AffineTransform::ConvertToProto() const {
+  AffineTransformData affine_transform_data;
+  ConvertToProto(&affine_transform_data);
+  return affine_transform_data;
+}
+
+bool compare(float lhs, float rhs, float epsilon = 0.001f) {
+  return std::fabs(lhs - rhs) < epsilon;
+}
+
+bool AffineTransform::Equals(const AffineTransform& other,
+                             float epsilon) const {
+  auto trans1 = GetTranslation();
+  auto trans2 = other.GetTranslation();
+
+  if (!(compare(trans1.x(), trans2.x(), epsilon) &&
+        compare(trans1.y(), trans2.y(), epsilon)))
+    return false;
+
+  auto scale1 = GetScale();
+  auto scale2 = other.GetScale();
+
+  if (!(compare(scale1.x(), scale2.x(), epsilon) &&
+        compare(scale1.y(), scale2.y(), epsilon)))
+    return false;
+
+  auto shear1 = GetShear();
+  auto shear2 = other.GetShear();
+
+  if (!(compare(shear1.x(), shear2.x(), epsilon) &&
+        compare(shear1.y(), shear2.y(), epsilon)))
+    return false;
+
+  auto rot1 = GetRotation();
+  auto rot2 = other.GetRotation();
+
+  if (!compare(rot1, rot2, epsilon)) {
+    return false;
+  }
+
+  return true;
+}
+
+bool AffineTransform::Equal(const AffineTransform& lhs,
+                            const AffineTransform& rhs, float epsilon) {
+  return lhs.Equals(rhs, epsilon);
+}
+
+MEDIAPIPE_REGISTER_TYPE(mediapipe::AffineTransform,
+                        "::mediapipe::AffineTransform", nullptr, nullptr);
+
+}  // namespace mediapipe

mediapipe/framework/formats/affine_transform.h

@@ -0,0 +1,86 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A container for affine transform data
+// This wrapper provides two functionalities:
+//  1. Factory methods for creation of Transform objects and thus
+//     AffineTransformData protocol buffers. These methods guarantee a valid
+//     affine transform data and are the preferred way of creating such.
+//  2. Accessors which allow for access of the data and the convertion to proto
+//     format
+
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AFFINE_TRANSFORM_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_AFFINE_TRANSFORM_H_
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/framework/formats/affine_transform_data.pb.h"
+#include "mediapipe/framework/port.h"
+#include "mediapipe/framework/port/point2.h"
+
+namespace mediapipe {
+
+class AffineTransform {
+ public:
+  // CREATION METHODS.
+  AffineTransform();
+
+  // Constructs a affine transform wrapping the specified affine transform data.
+  // Checks the validity of the input and crashes upon failure.
+  explicit AffineTransform(const AffineTransformData& transform_data);
+
+  static AffineTransform Create(const Point2_f& translation = Point2_f(0, 0),
+                                const Point2_f& scale = Point2_f(1, 1),
+                                float rotation = 0,
+                                const Point2_f& shear = Point2_f(0, 0));
+
+  // ACCESSORS
+  // Accessor for the composition matrix
+  std::vector<float> GetCompositionMatrix();
+
+  Point2_f GetScale() const;
+  Point2_f GetTranslation() const;
+  Point2_f GetShear() const;
+  float GetRotation() const;
+
+  void SetScale(const Point2_f& scale);
+  void SetTranslation(const Point2_f& translation);
+  void SetShear(const Point2_f& shear);
+  void SetRotation(float rotation);
+
+  void AddScale(const Point2_f& scale);
+  void AddTranslation(const Point2_f& translation);
+  void AddShear(const Point2_f& shear);
+  void AddRotation(float rotation);
+
+  // Serializes and deserializes the affine transform object.
+  void ConvertToProto(AffineTransformData* proto) const;
+  AffineTransformData ConvertToProto() const;
+  void SetFromProto(const AffineTransformData& proto);
+
+  bool Equals(const AffineTransform& other, float epsilon = 0.001f) const;
+
+  static bool Equal(const AffineTransform& lhs, const AffineTransform& rhs,
+                    float epsilon = 0.001f);
+
+ private:
+  // The wrapped transform data.
+  AffineTransformData affine_transform_data_;
+  std::vector<float> matrix_ = {1, 0, 0, 0, 1, 0, 0, 0, 1};
+  bool is_dirty_ = false;
+};
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_AFFINE_TRANSFORM_H_

mediapipe/framework/formats/affine_transform_data.proto

@@ -0,0 +1,33 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+option objc_class_prefix = "MediaPipe";
+
+// Proto for serializing Vector2 data
+message Vector2Data {
+  optional float x = 1;
+  optional float y = 2;
+}
+
+// Proto for serializing Affine Transform data.
+message AffineTransformData {
+  optional Vector2Data translation = 1;
+  optional Vector2Data scale = 2;
+  optional Vector2Data shear = 3;
+  optional float rotation = 4;  // in radians
+}

mediapipe/framework/formats/affine_transform_test.cc

@@ -0,0 +1,94 @@
+#include "mediapipe/framework/formats/affine_transform.h"
+
+#include <string>
+
+#include "base/logging.h"
+#include "mediapipe/framework/formats/affine_transform_data.pb.h"
+#include "mediapipe/framework/port/point2.h"
+#include "testing/base/public/gmock.h"
+#include "testing/base/public/gunit.h"
+
+namespace mediapipe {
+
+TEST(AffineTransformTest, TraslationTest) {
+  AffineTransform transform;
+  transform.SetTranslation(Point2_f(10, -3));
+
+  auto trans = transform.GetTranslation();
+  EXPECT_FLOAT_EQ(10, trans.x());
+  EXPECT_FLOAT_EQ(-3, trans.y());
+
+  transform.AddTranslation(Point2_f(-10, 3));
+
+  trans = transform.GetTranslation();
+  EXPECT_FLOAT_EQ(0, trans.x());
+  EXPECT_FLOAT_EQ(0, trans.y());
+}
+
+TEST(AffineTransformTest, ScaleTest) {
+  AffineTransform transform;
+  transform.SetScale(Point2_f(10, -3));
+
+  auto scale = transform.GetScale();
+  EXPECT_FLOAT_EQ(10, scale.x());
+  EXPECT_FLOAT_EQ(-3, scale.y());
+
+  transform.AddScale(Point2_f(-10, 3));
+
+  scale = transform.GetScale();
+  EXPECT_FLOAT_EQ(0, scale.x());
+  EXPECT_FLOAT_EQ(0, scale.y());
+}
+
+TEST(AffineTransformTest, RotationTest) {
+  AffineTransform transform;
+  transform.SetRotation(0.7);
+
+  float rot = transform.GetRotation();
+  EXPECT_FLOAT_EQ(0.7, rot);
+
+  transform.AddRotation(-0.7);
+  rot = transform.GetRotation();
+  EXPECT_FLOAT_EQ(0, rot);
+}
+
+TEST(AffineTransformTest, ShearTest) {
+  AffineTransform transform;
+  transform.SetShear(Point2_f(10, -3));
+
+  auto shear = transform.GetShear();
+  EXPECT_FLOAT_EQ(10, shear.x());
+  EXPECT_FLOAT_EQ(-3, shear.y());
+
+  transform.AddShear(Point2_f(-10, 3));
+
+  shear = transform.GetShear();
+  EXPECT_FLOAT_EQ(0, shear.x());
+  EXPECT_FLOAT_EQ(0, shear.y());
+}
+
+TEST(AffineTransformTest, TransformTest) {
+  AffineTransform transform1;
+  transform1 = AffineTransform::Create(Point2_f(0.1, -0.2), Point2_f(0.3, -0.4),
+                                       0.5, Point2_f(0.6, -0.7));
+
+  AffineTransform transform2;
+  transform2 = AffineTransform::Create(Point2_f(0.1, -0.2), Point2_f(0.3, -0.4),
+                                       0.5, Point2_f(0.6, -0.7));
+
+  EXPECT_THAT(true, transform1.Equals(transform2));
+  EXPECT_THAT(true, AffineTransform::Equal(transform1, transform2));
+
+  transform1 = AffineTransform::Create(Point2_f(0.00001, -0.00002),
+                                       Point2_f(0.00003, -0.00004), 0.00005,
+                                       Point2_f(0.00006, -0.00007));
+
+  transform2 = AffineTransform::Create(Point2_f(0.00001, -0.00002),
+                                       Point2_f(0.00003, -0.00004), 0.00005,
+                                       Point2_f(0.00006, -0.00007));
+
+  EXPECT_THAT(true, transform1.Equals(transform2, 0.000001));
+  EXPECT_THAT(true, AffineTransform::Equal(transform1, transform2, 0.000001));
+}
+
+}  // namespace mediapipe

mediapipe/framework/formats/image_multi_pool.cc

@@ -23,6 +23,7 @@
 #if !MEDIAPIPE_DISABLE_GPU
 #ifdef __APPLE__
 #include "mediapipe/objc/CFHolder.h"
+#include "mediapipe/objc/util.h"
 #endif  // __APPLE__
 #endif  // !MEDIAPIPE_DISABLE_GPU
 

mediapipe/framework/graph_output_stream.cc

@@ -125,9 +125,10 @@ absl::Status OutputStreamObserver::Notify() {
 absl::Status OutputStreamPollerImpl::Initialize(
     const std::string& stream_name, const PacketType* packet_type,
     std::function<void(InputStreamManager*, bool*)> queue_size_callback,
-    OutputStreamManager* output_stream_manager) {
+    OutputStreamManager* output_stream_manager, bool observe_timestamp_bounds) {
   MP_RETURN_IF_ERROR(GraphOutputStream::Initialize(stream_name, packet_type,
-                                                   output_stream_manager));
+                                                   output_stream_manager,
+                                                   observe_timestamp_bounds));
   input_stream_handler_->SetQueueSizeCallbacks(queue_size_callback,
                                                queue_size_callback);
   return absl::OkStatus();
@@ -176,11 +177,17 @@ void OutputStreamPollerImpl::NotifyError() {
 bool OutputStreamPollerImpl::Next(Packet* packet) {
   CHECK(packet);
   bool empty_queue = true;
+  bool timestamp_bound_changed = false;
   Timestamp min_timestamp = Timestamp::Unset();
   mutex_.Lock();
   while (true) {
     min_timestamp = input_stream_->MinTimestampOrBound(&empty_queue);
-    if (graph_has_error_ || !empty_queue ||
+    if (empty_queue) {
+      timestamp_bound_changed =
+          input_stream_handler_->ProcessTimestampBounds() &&
+          output_timestamp_ < min_timestamp.PreviousAllowedInStream();
+    }
+    if (graph_has_error_ || !empty_queue || timestamp_bound_changed ||
         min_timestamp == Timestamp::Done()) {
       break;
     } else {
@@ -191,17 +198,26 @@ bool OutputStreamPollerImpl::Next(Packet* packet) {
     mutex_.Unlock();
     return false;
   }
+  if (empty_queue) {
+    output_timestamp_ = min_timestamp.PreviousAllowedInStream();
+  } else {
+    output_timestamp_ = min_timestamp;
+  }
   mutex_.Unlock();
   if (min_timestamp == Timestamp::Done()) {
     return false;
   }
-  int num_packets_dropped = 0;
-  bool stream_is_done = false;
-  *packet = input_stream_->PopPacketAtTimestamp(
-      min_timestamp, &num_packets_dropped, &stream_is_done);
-  CHECK_EQ(num_packets_dropped, 0)
-      << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".",
-                          num_packets_dropped, input_stream_->Name());
+  if (!empty_queue) {
+    int num_packets_dropped = 0;
+    bool stream_is_done = false;
+    *packet = input_stream_->PopPacketAtTimestamp(
+        min_timestamp, &num_packets_dropped, &stream_is_done);
+    CHECK_EQ(num_packets_dropped, 0)
+        << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".",
+                            num_packets_dropped, input_stream_->Name());
+  } else if (timestamp_bound_changed) {
+    *packet = Packet().At(min_timestamp.PreviousAllowedInStream());
+  }
   return true;
 }
 

mediapipe/framework/graph_output_stream.h

@@ -143,7 +143,8 @@ class OutputStreamPollerImpl : public GraphOutputStream {
   absl::Status Initialize(
       const std::string& stream_name, const PacketType* packet_type,
       std::function<void(InputStreamManager*, bool*)> queue_size_callback,
-      OutputStreamManager* output_stream_manager);
+      OutputStreamManager* output_stream_manager,
+      bool observe_timestamp_bounds = false);
 
   void PrepareForRun(std::function<void()> notification_callback,
                      std::function<void(absl::Status)> error_callback) override;
@@ -170,6 +171,7 @@ class OutputStreamPollerImpl : public GraphOutputStream {
   absl::Mutex mutex_;
   absl::CondVar handler_condvar_ ABSL_GUARDED_BY(mutex_);
   bool graph_has_error_ ABSL_GUARDED_BY(mutex_);
+  Timestamp output_timestamp_ ABSL_GUARDED_BY(mutex_) = Timestamp::Min();
 };
 
 }  // namespace internal

mediapipe/framework/port/BUILD

@@ -89,6 +89,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "commandlineflags",
+    hdrs = [
+        "commandlineflags.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//third_party:glog",
+        "@com_google_absl//absl/flags:flag",
+    ],
+)
+
 cc_library(
     name = "core_proto",
     hdrs = [

mediapipe/framework/port/commandlineflags.h

@@ -0,0 +1,30 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PORT_COMMANDLINEFLAGS_H_
+#define MEDIAPIPE_PORT_COMMANDLINEFLAGS_H_
+
+#include "gflags/gflags.h"
+namespace absl {
+template <typename T>
+T GetFlag(const T& f) {
+  return f;
+}
+template <typename T, typename U>
+void SetFlag(T* f, const U& u) {
+  *f = u;
+}
+}  // namespace absl
+
+#endif  // MEDIAPIPE_PORT_COMMANDLINEFLAGS_H_

mediapipe/framework/testdata/night_light_calculator.proto

@@ -51,4 +51,17 @@ message NightLightCalculatorOptions {
 
   // Format string used by string::Substitute to construct the output.
   optional string format_string = 9;
+
+  message LightBundle {
+    optional string room_id = 1;
+    repeated NightLightCalculatorOptions room_lights = 2;
+  }
+
+  repeated LightBundle bundle = 10;
+
+  // The number of night-lights.
+  repeated int32 num_lights = 11;
+
+  // Options for nested night-lights.
+  optional NightLightCalculatorOptions sub_options = 12;
 }

mediapipe/framework/tool/BUILD

@@ -180,15 +180,67 @@ cc_library(
     ],
 )
 
+mediapipe_proto_library(
+    name = "field_data_proto",
+    srcs = ["field_data.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["@com_google_protobuf//:any_proto"],
+)
+
+cc_library(
+    name = "options_field_util",
+    srcs = ["options_field_util.cc"],
+    hdrs = ["options_field_util.h"],
+    visibility = ["//mediapipe/framework:mediapipe_internal"],
+    deps = [
+        ":field_data_cc_proto",
+        ":name_util",
+        ":options_registry",
+        ":proto_util_lite",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:packet_type",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:any_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "options_syntax_util",
+    srcs = ["options_syntax_util.cc"],
+    hdrs = ["options_syntax_util.h"],
+    visibility = ["//mediapipe/framework:mediapipe_internal"],
+    deps = [
+        ":name_util",
+        ":options_field_util",
+        ":options_registry",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:packet_type",
+        "//mediapipe/framework/port:advanced_proto",
+        "//mediapipe/framework/port:any_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "options_util",
     srcs = ["options_util.cc"],
     hdrs = ["options_util.h"],
     visibility = ["//mediapipe/framework:mediapipe_internal"],
     deps = [
+        ":options_field_util",
         ":options_map",
+        ":options_registry",
+        ":options_syntax_util",
         ":proto_util_lite",
         "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:collection",
         "//mediapipe/framework:input_stream_shard",
         "//mediapipe/framework:output_side_packet",
@@ -199,7 +251,7 @@ cc_library(
         "//mediapipe/framework/port:advanced_proto",
         "//mediapipe/framework/port:any_proto",
         "//mediapipe/framework/port:status",
-        "//mediapipe/framework/tool:type_util",
+        "//mediapipe/framework/tool:name_util",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -227,6 +279,8 @@ cc_library(
         "//mediapipe/framework/deps:registration",
         "//mediapipe/framework/port:advanced_proto",
         "//mediapipe/framework/port:logging",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/synchronization",
     ],
 )
 
@@ -238,7 +292,9 @@ mediapipe_cc_test(
     data = [":node_chain_subgraph.proto"],
     requires_full_emulation = False,
     deps = [
+        ":options_field_util",
         ":options_registry",
+        ":options_syntax_util",
         ":options_util",
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/core:flow_limiter_calculator_cc_proto",
@@ -246,12 +302,14 @@ mediapipe_cc_test(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:validated_graph_config",
+        "//mediapipe/framework/deps:message_matchers",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/testdata:night_light_calculator_options_lib",
         "//mediapipe/framework/tool:node_chain_subgraph_options_lib",
         "//mediapipe/util:header_util",
+        "@com_google_absl//absl/strings",
     ],
 )
 

mediapipe/framework/tool/field_data.proto

@@ -0,0 +1,47 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Forked from mediapipe/framework/tool/source.proto.
+// The forked proto must remain identical to the original proto and should be
+// ONLY used by mediapipe open source project.
+
+syntax = "proto2";
+
+package mediapipe;
+
+// `MessageData`, like protobuf.Any, contains an arbitrary serialized protbuf
+// along with a URL that describes the type of the serialized message.
+message MessageData {
+  // A URL/resource name that identifies the type of serialized protbuf.
+  optional string type_url = 1;
+
+  // Must be a valid serialized protocol buffer of the above specified type.
+  optional bytes value = 2;
+}
+
+// Data for one Protobuf field or one MediaPipe packet.
+message FieldData {
+  oneof value {
+    sint32 int32_value = 1;
+    sint64 int64_value = 2;
+    uint32 uint32_value = 3;
+    uint64 uint64_value = 4;
+    double double_value = 5;
+    float float_value = 6;
+    bool bool_value = 7;
+    sint32 enum_value = 8;
+    string string_value = 9;
+    MessageData message_value = 10;
+  }
+}

mediapipe/framework/tool/options_field_util.cc

@@ -0,0 +1,698 @@
+
+#include "mediapipe/framework/tool/options_field_util.h"
+
+#include <memory>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/packet_type.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/any_proto.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/name_util.h"
+#include "mediapipe/framework/tool/proto_util_lite.h"
+
+namespace mediapipe {
+namespace tool {
+namespace options_field_util {
+
+using ::mediapipe::proto_ns::internal::WireFormatLite;
+using FieldType = WireFormatLite::FieldType;
+using ::mediapipe::proto_ns::io::ArrayInputStream;
+using ::mediapipe::proto_ns::io::CodedInputStream;
+using ::mediapipe::proto_ns::io::CodedOutputStream;
+using ::mediapipe::proto_ns::io::StringOutputStream;
+
+// Utility functions for OptionsFieldUtil.
+namespace {
+
+// The type name for the proto3 "Any" type.
+constexpr absl::string_view kGoogleProtobufAny = "google.protobuf.Any";
+
+// Converts a FieldDescriptor::Type to the corresponding FieldType.
+FieldType AsFieldType(proto_ns::FieldDescriptorProto::Type type) {
+  return static_cast<FieldType>(type);
+}
+
+absl::Status WriteValue(const FieldData& value, FieldType field_type,
+                        std::string* field_bytes) {
+  StringOutputStream sos(field_bytes);
+  CodedOutputStream out(&sos);
+  switch (field_type) {
+    case WireFormatLite::TYPE_INT32:
+      WireFormatLite::WriteInt32NoTag(value.int32_value(), &out);
+      break;
+    case WireFormatLite::TYPE_SINT32:
+      WireFormatLite::WriteSInt32NoTag(value.int32_value(), &out);
+      break;
+    case WireFormatLite::TYPE_INT64:
+      WireFormatLite::WriteInt64NoTag(value.int64_value(), &out);
+      break;
+    case WireFormatLite::TYPE_SINT64:
+      WireFormatLite::WriteSInt64NoTag(value.int64_value(), &out);
+      break;
+    case WireFormatLite::TYPE_UINT32:
+      WireFormatLite::WriteUInt32NoTag(value.uint32_value(), &out);
+      break;
+    case WireFormatLite::TYPE_UINT64:
+      WireFormatLite::WriteUInt64NoTag(value.uint64_value(), &out);
+      break;
+    case WireFormatLite::TYPE_DOUBLE:
+      WireFormatLite::WriteDoubleNoTag(value.uint64_value(), &out);
+      break;
+    case WireFormatLite::TYPE_FLOAT:
+      WireFormatLite::WriteFloatNoTag(value.float_value(), &out);
+      break;
+    case WireFormatLite::TYPE_BOOL:
+      WireFormatLite::WriteBoolNoTag(value.bool_value(), &out);
+      break;
+    case WireFormatLite::TYPE_ENUM:
+      WireFormatLite::WriteEnumNoTag(value.enum_value(), &out);
+      break;
+    case WireFormatLite::TYPE_STRING:
+      out.WriteString(value.string_value());
+      break;
+    case WireFormatLite::TYPE_MESSAGE:
+      out.WriteString(value.message_value().value());
+      break;
+    default:
+      return absl::UnimplementedError(
+          absl::StrCat("Cannot write type: ", field_type));
+  }
+  return absl::OkStatus();
+}
+
+// Serializes a packet value.
+absl::Status WriteField(const FieldData& packet, const FieldDescriptor* field,
+                        std::string* result) {
+  FieldType field_type = AsFieldType(field->type());
+  return WriteValue(packet, field_type, result);
+}
+
+template <typename ValueT, FieldType kFieldType>
+static ValueT ReadValue(absl::string_view field_bytes, absl::Status* status) {
+  ArrayInputStream ais(field_bytes.data(), field_bytes.size());
+  CodedInputStream input(&ais);
+  ValueT result;
+  if (!WireFormatLite::ReadPrimitive<ValueT, kFieldType>(&input, &result)) {
+    status->Update(mediapipe::InvalidArgumentError(absl::StrCat(
+        "Bad serialized value: ", MediaPipeTypeStringOrDemangled<ValueT>(),
+        ".")));
+  }
+  return result;
+}
+
+absl::Status ReadValue(absl::string_view field_bytes, FieldType field_type,
+                       absl::string_view message_type, FieldData* result) {
+  absl::Status status;
+  result->Clear();
+  switch (field_type) {
+    case WireFormatLite::TYPE_INT32:
+      result->set_int32_value(
+          ReadValue<int32, WireFormatLite::TYPE_INT32>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_SINT32:
+      result->set_int32_value(
+          ReadValue<int32, WireFormatLite::TYPE_SINT32>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_INT64:
+      result->set_int64_value(
+          ReadValue<int64, WireFormatLite::TYPE_INT64>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_SINT64:
+      result->set_int64_value(
+          ReadValue<int64, WireFormatLite::TYPE_SINT64>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_UINT32:
+      result->set_uint32_value(
+          ReadValue<uint32, WireFormatLite::TYPE_UINT32>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_UINT64:
+      result->set_uint64_value(
+          ReadValue<uint32, WireFormatLite::TYPE_UINT32>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_DOUBLE:
+      result->set_double_value(
+          ReadValue<double, WireFormatLite::TYPE_DOUBLE>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_FLOAT:
+      result->set_float_value(
+          ReadValue<float, WireFormatLite::TYPE_FLOAT>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_BOOL:
+      result->set_bool_value(
+          ReadValue<bool, WireFormatLite::TYPE_BOOL>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_ENUM:
+      result->set_enum_value(
+          ReadValue<int32, WireFormatLite::TYPE_ENUM>(field_bytes, &status));
+      break;
+    case WireFormatLite::TYPE_STRING:
+      result->set_string_value(std::string(field_bytes));
+      break;
+    case WireFormatLite::TYPE_MESSAGE:
+      result->mutable_message_value()->set_value(std::string(field_bytes));
+      result->mutable_message_value()->set_type_url(TypeUrl(message_type));
+      break;
+    default:
+      status = absl::UnimplementedError(
+          absl::StrCat("Cannot read type: ", field_type));
+      break;
+  }
+  return status;
+}
+
+// Deserializes a packet from a protobuf field.
+absl::Status ReadField(absl::string_view bytes, const FieldDescriptor* field,
+                       FieldData* result) {
+  RET_CHECK_NE(field, nullptr);
+  FieldType field_type = AsFieldType(field->type());
+  std::string message_type = (field_type == WireFormatLite::TYPE_MESSAGE)
+                                 ? field->message_type()->full_name()
+                                 : "";
+  return ReadValue(bytes, field_type, message_type, result);
+}
+
+// Reads all values from a repeated field.
+absl::Status GetFieldValues(const FieldData& message_data,
+                            const FieldDescriptor& field,
+                            std::vector<FieldData>* result) {
+  const std::string& message_bytes = message_data.message_value().value();
+  FieldType field_type = AsFieldType(field.type());
+  ProtoUtilLite proto_util;
+  ProtoUtilLite::ProtoPath proto_path = {{field.number(), 0}};
+  int count;
+  MP_RETURN_IF_ERROR(
+      proto_util.GetFieldCount(message_bytes, proto_path, field_type, &count));
+  std::vector<std::string> field_values;
+  MP_RETURN_IF_ERROR(proto_util.GetFieldRange(message_bytes, proto_path, count,
+                                              field_type, &field_values));
+  for (int i = 0; i < count; ++i) {
+    FieldData r;
+    MP_RETURN_IF_ERROR(ReadField(field_values[i], &field, &r));
+    result->push_back(std::move(r));
+  }
+  return absl::OkStatus();
+}
+
+// Reads one value from a field.
+absl::Status GetFieldValue(const FieldData& message_data,
+                           const FieldPathEntry& entry, FieldData* result) {
+  RET_CHECK_NE(entry.field, nullptr);
+  const std::string& message_bytes = message_data.message_value().value();
+  FieldType field_type = AsFieldType(entry.field->type());
+  ProtoUtilLite proto_util;
+  ProtoUtilLite::ProtoPath proto_path = {{entry.field->number(), entry.index}};
+  std::vector<std::string> field_values;
+  MP_RETURN_IF_ERROR(proto_util.GetFieldRange(message_bytes, proto_path, 1,
+                                              field_type, &field_values));
+  MP_RETURN_IF_ERROR(ReadField(field_values[0], entry.field, result));
+  return absl::OkStatus();
+}
+
+// Writes one value to a field.
+absl::Status SetFieldValue(const FieldPathEntry& entry, const FieldData& value,
+                           FieldData* result) {
+  std::vector<FieldData> field_values;
+  ProtoUtilLite proto_util;
+  FieldType field_type = AsFieldType(entry.field->type());
+  ProtoUtilLite::ProtoPath proto_path = {{entry.field->number(), entry.index}};
+  std::string* message_bytes = result->mutable_message_value()->mutable_value();
+  int field_count;
+  MP_RETURN_IF_ERROR(proto_util.GetFieldCount(*message_bytes, proto_path,
+                                              field_type, &field_count));
+  if (entry.index > field_count) {
+    return absl::OutOfRangeError(
+        absl::StrCat("Option field index out of range: ", entry.index));
+  }
+  int replace_length = entry.index < field_count ? 1 : 0;
+  std::string field_value;
+  MP_RETURN_IF_ERROR(WriteField(value, entry.field, &field_value));
+  MP_RETURN_IF_ERROR(proto_util.ReplaceFieldRange(
+      message_bytes, proto_path, replace_length, field_type, {field_value}));
+  return absl::OkStatus();
+}
+
+// Returns true for a field of type "google.protobuf.Any".
+bool IsProtobufAny(const FieldDescriptor* field) {
+  return AsFieldType(field->type()) == FieldType::TYPE_MESSAGE &&
+         field->message_type()->full_name() == kGoogleProtobufAny;
+}
+
+// Returns the message FieldData from a serialized protobuf.Any.
+FieldData ParseProtobufAny(const FieldData& data) {
+  protobuf::Any any;
+  any.ParseFromString(data.message_value().value());
+  FieldData result;
+  result.mutable_message_value()->set_value(std::string(any.value()));
+  result.mutable_message_value()->set_type_url(any.type_url());
+  return result;
+}
+
+// Returns the serialized protobuf.Any containing a message FieldData.
+FieldData SerializeProtobufAny(const FieldData& data) {
+  protobuf::Any any;
+  any.set_value(data.message_value().value());
+  any.set_type_url(data.message_value().type_url());
+  FieldData result;
+  result.mutable_message_value()->set_value(any.SerializeAsString());
+  result.mutable_message_value()->set_type_url(TypeUrl(kGoogleProtobufAny));
+  return result;
+}
+
+// Returns the field index of an extension type in a repeated field.
+StatusOr<int> FindExtensionIndex(const FieldData& message_data,
+                                 FieldPathEntry* entry) {
+  if (entry->field == nullptr || !IsProtobufAny(entry->field)) {
+    return -1;
+  }
+  std::string& extension_type = entry->extension_type;
+  std::vector<FieldData> field_values;
+  RET_CHECK_NE(entry->field, nullptr);
+  MP_RETURN_IF_ERROR(
+      GetFieldValues(message_data, *entry->field, &field_values));
+  for (int i = 0; i < field_values.size(); ++i) {
+    FieldData extension = ParseProtobufAny(field_values[i]);
+    if (extension_type == "*" ||
+        ParseTypeUrl(extension.message_value().type_url()) == extension_type) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+// Returns true if the value of a field is available.
+bool HasField(const FieldPath& field_path, const FieldData& message_data) {
+  FieldData value;
+  return GetField(field_path, message_data, &value).ok() &&
+         value.value_case() != mediapipe::FieldData::VALUE_NOT_SET;
+}
+
+// Returns the extension field containing the specified extension-type.
+const FieldDescriptor* FindExtensionField(const FieldData& message_data,
+                                          absl::string_view extension_type) {
+  std::string message_type =
+      ParseTypeUrl(message_data.message_value().type_url());
+  std::vector<const FieldDescriptor*> extensions;
+  OptionsRegistry::FindAllExtensions(message_type, &extensions);
+  for (const FieldDescriptor* extension : extensions) {
+    if (extension->message_type()->full_name() == extension_type) {
+      return extension;
+    }
+    if (extension_type == "*" && HasField({{extension, 0}}, message_data)) {
+      return extension;
+    }
+  }
+  return nullptr;
+}
+
+// Sets a protobuf in a repeated protobuf::Any field.
+void SetOptionsMessage(
+    const FieldData& node_options,
+    proto_ns::RepeatedPtrField<mediapipe::protobuf::Any>* result) {
+  protobuf::Any* options_any = nullptr;
+  for (auto& any : *result) {
+    if (any.type_url() == node_options.message_value().type_url()) {
+      options_any = &any;
+    }
+  }
+  if (!options_any) {
+    options_any = result->Add();
+    options_any->set_type_url(node_options.message_value().type_url());
+  }
+  *options_any->mutable_value() = node_options.message_value().value();
+}
+
+// Returns the count of values in a repeated field.
+int FieldCount(const FieldData& message_data, const FieldDescriptor* field) {
+  const std::string& message_bytes = message_data.message_value().value();
+  FieldType field_type = AsFieldType(field->type());
+  ProtoUtilLite proto_util;
+  ProtoUtilLite::ProtoPath proto_path = {{field->number(), 0}};
+  int count;
+  if (proto_util.GetFieldCount(message_bytes, proto_path, field_type, &count)
+          .ok()) {
+    return count;
+  }
+  return 0;
+}
+
+}  // anonymous namespace
+
+// Deserializes a packet containing a MessageLite value.
+absl::Status ReadMessage(const std::string& value, const std::string& type_name,
+                         Packet* result) {
+  auto packet = packet_internal::PacketFromDynamicProto(type_name, value);
+  if (packet.ok()) {
+    *result = *packet;
+  }
+  return packet.status();
+}
+
+// Merge two options FieldData values.
+absl::Status MergeMessages(const FieldData& base, const FieldData& over,
+                           FieldData* result) {
+  absl::Status status;
+  if (over.value_case() == FieldData::VALUE_NOT_SET) {
+    *result = base;
+    return status;
+  }
+  if (base.value_case() == FieldData::VALUE_NOT_SET) {
+    *result = over;
+    return status;
+  }
+  if (over.value_case() != base.value_case()) {
+    return absl::InvalidArgumentError(absl::StrCat(
+        "Cannot merge field data with data types: ", base.value_case(), ", ",
+        over.value_case()));
+  }
+  if (over.message_value().type_url() != base.message_value().type_url()) {
+    return absl::InvalidArgumentError(
+        absl::StrCat("Cannot merge field data with message types: ",
+                     base.message_value().type_url(), ", ",
+                     over.message_value().type_url()));
+  }
+  absl::Cord merged_value;
+  merged_value.Append(base.message_value().value());
+  merged_value.Append(over.message_value().value());
+  result->mutable_message_value()->set_type_url(
+      base.message_value().type_url());
+  result->mutable_message_value()->set_value(std::string(merged_value));
+  return status;
+}
+
+// Returns either the extension field or the repeated protobuf.Any field index
+// holding the specified extension-type.
+absl::Status FindExtension(const FieldData& message_data,
+                           FieldPathEntry* entry) {
+  if (entry->extension_type.empty()) {
+    return absl::OkStatus();
+  }
+
+  // For repeated protobuf::Any, find the index for the extension_type.
+  ASSIGN_OR_RETURN(int index, FindExtensionIndex(message_data, entry));
+  if (index != -1) {
+    entry->index = index;
+    return absl::OkStatus();
+  }
+
+  // Returns the extension field containing the specified extension-type.
+  std::string& extension_type = entry->extension_type;
+  const FieldDescriptor* field =
+      FindExtensionField(message_data, extension_type);
+  if (field != nullptr) {
+    entry->field = field;
+    entry->index = 0;
+    return absl::OkStatus();
+  }
+  return absl::NotFoundError(
+      absl::StrCat("Option extension not found: ", extension_type));
+}
+
+// Return the FieldPath referencing an extension message.
+FieldPath GetExtensionPath(const std::string& parent_type,
+                           const std::string& extension_type,
+                           const std::string& field_name,
+                           bool is_protobuf_any) {
+  FieldPath result;
+  const tool::Descriptor* parent_descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(parent_type);
+  FieldPathEntry field_entry;
+  field_entry.field = parent_descriptor->FindFieldByName(field_name);
+  if (is_protobuf_any) {
+    field_entry.extension_type = extension_type;
+    result = {std::move(field_entry)};
+  } else {
+    field_entry.index = 0;
+    FieldPathEntry extension_entry;
+    extension_entry.extension_type = extension_type;
+    result = {std::move(field_entry), std::move(extension_entry)};
+  }
+  return result;
+}
+
+// Returns the requested options protobuf for a graph node.
+absl::Status GetNodeOptions(const FieldData& message_data,
+                            const std::string& extension_type,
+                            FieldData* result) {
+  constexpr char kOptionsName[] = "options";
+  constexpr char kNodeOptionsName[] = "node_options";
+  std::string parent_type = options_field_util::ParseTypeUrl(
+      std::string(message_data.message_value().type_url()));
+  FieldPath path;
+  Status status;
+  path = GetExtensionPath(parent_type, extension_type, kOptionsName, false);
+  status = GetField(path, message_data, result);
+  if (status.ok()) {
+    return status;
+  }
+  path = GetExtensionPath(parent_type, extension_type, kNodeOptionsName, true);
+  status = GetField(path, message_data, result);
+  return status;
+}
+
+// Returns the requested options protobuf for a graph.
+absl::Status GetGraphOptions(const FieldData& message_data,
+                             const std::string& extension_type,
+                             FieldData* result) {
+  constexpr char kOptionsName[] = "options";
+  constexpr char kGraphOptionsName[] = "graph_options";
+  std::string parent_type = options_field_util::ParseTypeUrl(
+      std::string(message_data.message_value().type_url()));
+  FieldPath path;
+  Status status;
+  path = GetExtensionPath(parent_type, extension_type, kOptionsName, false);
+  status = GetField(path, message_data, result);
+  if (status.ok()) {
+    return status;
+  }
+  path = GetExtensionPath(parent_type, extension_type, kGraphOptionsName, true);
+  status = GetField(path, message_data, result);
+  return status;
+}
+
+// Reads a FieldData value from a protobuf field.
+absl::Status GetField(const FieldPath& field_path,
+                      const FieldData& message_data, FieldData* result) {
+  if (field_path.empty()) {
+    *result->mutable_message_value() = message_data.message_value();
+    return absl::OkStatus();
+  }
+  FieldPathEntry head = field_path.front();
+  FieldPath tail = field_path;
+  tail.erase(tail.begin());
+  if (!head.extension_type.empty()) {
+    MP_RETURN_IF_ERROR(FindExtension(message_data, &head));
+  }
+  if (tail.empty() && FieldCount(message_data, head.field) == 0) {
+    return absl::OkStatus();
+  }
+  MP_RETURN_IF_ERROR(GetFieldValue(message_data, head, result));
+  if (IsProtobufAny(head.field)) {
+    *result = ParseProtobufAny(*result);
+  }
+  if (!tail.empty()) {
+    FieldData child = *result;
+    MP_RETURN_IF_ERROR(GetField(tail, child, result));
+  }
+  return absl::OkStatus();
+}
+
+// Writes a FieldData value into protobuf field.
+absl::Status SetField(const FieldPath& field_path, const FieldData& value,
+                      FieldData* message_data) {
+  if (field_path.empty()) {
+    *message_data->mutable_message_value() = value.message_value();
+    return absl::OkStatus();
+  }
+  FieldPathEntry head = field_path.front();
+  FieldPath tail = field_path;
+  tail.erase(tail.begin());
+  if (!head.extension_type.empty()) {
+    MP_RETURN_IF_ERROR(FindExtension(*message_data, &head));
+  }
+  if (tail.empty()) {
+    MP_RETURN_IF_ERROR(SetFieldValue(head, value, message_data));
+  } else {
+    FieldData child;
+    MP_RETURN_IF_ERROR(GetFieldValue(*message_data, head, &child));
+    MP_RETURN_IF_ERROR(SetField(tail, value, &child));
+    if (IsProtobufAny(head.field)) {
+      child = SerializeProtobufAny(child);
+    }
+    MP_RETURN_IF_ERROR(SetFieldValue(head, child, message_data));
+  }
+  return absl::OkStatus();
+}
+
+// Merges a packet value into nested protobuf Message.
+absl::Status MergeField(const FieldPath& field_path, const FieldData& value,
+                        FieldData* message_data) {
+  absl::Status status;
+  FieldType field_type = field_path.empty()
+                             ? FieldType::TYPE_MESSAGE
+                             : AsFieldType(field_path.back().field->type());
+  std::string message_type =
+      (value.has_message_value())
+          ? ParseTypeUrl(std::string(value.message_value().type_url()))
+          : "";
+  FieldData v = value;
+  if (field_type == FieldType::TYPE_MESSAGE) {
+    FieldData b;
+    status.Update(GetField(field_path, *message_data, &b));
+    status.Update(MergeMessages(b, v, &v));
+  }
+  status.Update(SetField(field_path, v, message_data));
+  return status;
+}
+
+// Sets the node_options field in a Node, and clears the options field.
+void SetOptionsMessage(const FieldData& node_options,
+                       CalculatorGraphConfig::Node* node) {
+  SetOptionsMessage(node_options, node->mutable_node_options());
+  node->clear_options();
+}
+
+// Serialize a MessageLite to a FieldData.
+FieldData AsFieldData(const proto_ns::MessageLite& message) {
+  FieldData result;
+  *result.mutable_message_value()->mutable_value() =
+      message.SerializePartialAsString();
+  *result.mutable_message_value()->mutable_type_url() =
+      TypeUrl(message.GetTypeName());
+  return result;
+}
+
+// Represents a protobuf enum value stored in a Packet.
+struct ProtoEnum {
+  ProtoEnum(int32 v) : value(v) {}
+  int32 value;
+};
+
+absl::Status AsPacket(const FieldData& data, Packet* result) {
+  switch (data.value_case()) {
+    case FieldData::ValueCase::kInt32Value:
+      *result = MakePacket<int32>(data.int32_value());
+      break;
+    case FieldData::ValueCase::kInt64Value:
+      *result = MakePacket<int64>(data.int64_value());
+      break;
+    case FieldData::ValueCase::kUint32Value:
+      *result = MakePacket<uint32>(data.uint32_value());
+      break;
+    case FieldData::ValueCase::kUint64Value:
+      *result = MakePacket<uint64>(data.uint64_value());
+      break;
+    case FieldData::ValueCase::kDoubleValue:
+      *result = MakePacket<double>(data.double_value());
+      break;
+    case FieldData::ValueCase::kFloatValue:
+      *result = MakePacket<float>(data.float_value());
+      break;
+    case FieldData::ValueCase::kBoolValue:
+      *result = MakePacket<bool>(data.bool_value());
+      break;
+    case FieldData::ValueCase::kEnumValue:
+      *result = MakePacket<ProtoEnum>(data.enum_value());
+      break;
+    case FieldData::ValueCase::kStringValue:
+      *result = MakePacket<std::string>(data.string_value());
+      break;
+    case FieldData::ValueCase::kMessageValue: {
+      auto r = packet_internal::PacketFromDynamicProto(
+          ParseTypeUrl(std::string(data.message_value().type_url())),
+          std::string(data.message_value().value()));
+      if (!r.ok()) {
+        return r.status();
+      }
+      *result = r.value();
+      break;
+    }
+    case FieldData::VALUE_NOT_SET:
+      *result = Packet();
+  }
+  return absl::OkStatus();
+}
+
+absl::Status AsFieldData(Packet packet, FieldData* result) {
+  static const auto* kTypeIds = new std::map<size_t, int32>{
+      {tool::GetTypeHash<int32>(), WireFormatLite::CPPTYPE_INT32},
+      {tool::GetTypeHash<int64>(), WireFormatLite::CPPTYPE_INT64},
+      {tool::GetTypeHash<uint32>(), WireFormatLite::CPPTYPE_UINT32},
+      {tool::GetTypeHash<uint64>(), WireFormatLite::CPPTYPE_UINT64},
+      {tool::GetTypeHash<double>(), WireFormatLite::CPPTYPE_DOUBLE},
+      {tool::GetTypeHash<float>(), WireFormatLite::CPPTYPE_FLOAT},
+      {tool::GetTypeHash<bool>(), WireFormatLite::CPPTYPE_BOOL},
+      {tool::GetTypeHash<ProtoEnum>(), WireFormatLite::CPPTYPE_ENUM},
+      {tool::GetTypeHash<std::string>(), WireFormatLite::CPPTYPE_STRING},
+  };
+
+  if (packet.ValidateAsProtoMessageLite().ok()) {
+    result->mutable_message_value()->set_value(
+        packet.GetProtoMessageLite().SerializeAsString());
+    result->mutable_message_value()->set_type_url(
+        TypeUrl(packet.GetProtoMessageLite().GetTypeName()));
+    return absl::OkStatus();
+  }
+
+  if (kTypeIds->count(packet.GetTypeId()) == 0) {
+    return absl::UnimplementedError(absl::StrCat(
+        "Cannot construct FieldData for: ", packet.DebugTypeName()));
+  }
+
+  switch (kTypeIds->at(packet.GetTypeId())) {
+    case WireFormatLite::CPPTYPE_INT32:
+      result->set_int32_value(packet.Get<int32>());
+      break;
+    case WireFormatLite::CPPTYPE_INT64:
+      result->set_int64_value(packet.Get<int64>());
+      break;
+    case WireFormatLite::CPPTYPE_UINT32:
+      result->set_uint32_value(packet.Get<uint32>());
+      break;
+    case WireFormatLite::CPPTYPE_UINT64:
+      result->set_uint64_value(packet.Get<uint64>());
+      break;
+    case WireFormatLite::CPPTYPE_DOUBLE:
+      result->set_double_value(packet.Get<double>());
+      break;
+    case WireFormatLite::CPPTYPE_FLOAT:
+      result->set_float_value(packet.Get<float>());
+      break;
+    case WireFormatLite::CPPTYPE_BOOL:
+      result->set_bool_value(packet.Get<bool>());
+      break;
+    case WireFormatLite::CPPTYPE_ENUM:
+      result->set_enum_value(packet.Get<ProtoEnum>().value);
+      break;
+    case WireFormatLite::CPPTYPE_STRING:
+      result->set_string_value(packet.Get<std::string>());
+      break;
+  }
+  return absl::OkStatus();
+}
+
+std::string TypeUrl(absl::string_view type_name) {
+  constexpr std::string_view kTypeUrlPrefix = "type.googleapis.com/";
+  return absl::StrCat(std::string(kTypeUrlPrefix), std::string(type_name));
+}
+
+std::string ParseTypeUrl(absl::string_view type_url) {
+  constexpr std::string_view kTypeUrlPrefix = "type.googleapis.com/";
+  if (std::string(type_url).rfind(kTypeUrlPrefix, 0) == 0) {
+    return std::string(
+        type_url.substr(kTypeUrlPrefix.length(), std::string::npos));
+  }
+  return std::string(type_url);
+}
+
+}  // namespace options_field_util
+}  // namespace tool
+}  // namespace mediapipe

mediapipe/framework/tool/options_field_util.h

@@ -0,0 +1,85 @@
+#ifndef MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_FIELD_UTIL_H_
+#define MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_FIELD_UTIL_H_
+
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/packet_type.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/tool/field_data.pb.h"
+#include "mediapipe/framework/tool/options_registry.h"
+
+namespace mediapipe {
+
+namespace tool {
+
+// Utility to read and write Packet data from protobuf fields.
+namespace options_field_util {
+
+// A protobuf field and index description.
+struct FieldPathEntry {
+  const FieldDescriptor* field = nullptr;
+  int index = -1;
+  std::string extension_type;
+};
+
+// A chain of nested protobuf fields and indexes.
+using FieldPath = std::vector<FieldPathEntry>;
+
+// Writes a field value into protobuf field.
+absl::Status SetField(const FieldPath& field_path, const FieldData& value,
+                      FieldData* message_data);
+
+// Reads a field value from a protobuf field.
+absl::Status GetField(const FieldPath& field_path,
+                      const FieldData& message_data, FieldData* result);
+
+// Merges a field value into nested protobuf Message.
+absl::Status MergeField(const FieldPath& field_path, const FieldData& value,
+                        FieldData* message_data);
+
+// Deserializes a packet containing a MessageLite value.
+absl::Status ReadMessage(const std::string& value, const std::string& type_name,
+                         Packet* result);
+
+// Merge two options protobuf field values.
+absl::Status MergeMessages(const FieldData& base, const FieldData& over,
+                           FieldData* result);
+
+// Returns the requested options protobuf for a graph.
+absl::Status GetNodeOptions(const FieldData& message_data,
+                            const std::string& extension_type,
+                            FieldData* result);
+
+// Returns the requested options protobuf for a graph node.
+absl::Status GetGraphOptions(const FieldData& message_data,
+                             const std::string& extension_type,
+                             FieldData* result);
+
+// Sets the node_options field in a Node, and clears the options field.
+void SetOptionsMessage(const FieldData& node_options,
+                       CalculatorGraphConfig::Node* node);
+
+// Serialize a MessageLite to a FieldData.
+FieldData AsFieldData(const proto_ns::MessageLite& message);
+
+// Constructs a Packet for a FieldData proto.
+absl::Status AsPacket(const FieldData& data, Packet* result);
+
+// Constructs a FieldData proto for a Packet.
+absl::Status AsFieldData(Packet packet, FieldData* result);
+
+// Returns the protobuf type-url for a protobuf type-name.
+std::string TypeUrl(absl::string_view type_name);
+
+// Returns the protobuf type-name for a protobuf type-url.
+std::string ParseTypeUrl(absl::string_view type_url);
+
+}  // namespace options_field_util
+}  // namespace tool
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_FIELD_UTIL_H_

mediapipe/framework/tool/options_registry.cc

@@ -1,47 +1,112 @@
 #include "mediapipe/framework/tool/options_registry.h"
 
+#include "absl/synchronization/mutex.h"
+
 namespace mediapipe {
 namespace tool {
 
-proto_ns::DescriptorPool* OptionsRegistry::options_descriptor_pool() {
-  static proto_ns::DescriptorPool* result = new proto_ns::DescriptorPool();
-  return result;
+namespace {
+
+// Returns a canonical message type name, with any leading "." removed.
+std::string CanonicalTypeName(const std::string& type_name) {
+  return (type_name.rfind('.', 0) == 0) ? type_name.substr(1) : type_name;
 }
 
+}  // namespace
+
 RegistrationToken OptionsRegistry::Register(
     const proto_ns::FileDescriptorSet& files) {
+  absl::MutexLock lock(&mutex());
   for (auto& file : files.file()) {
-    options_descriptor_pool()->BuildFile(file);
+    for (auto& message_type : file.message_type()) {
+      Register(message_type, file.package());
+    }
   }
   return RegistrationToken([]() {});
 }
 
-const proto_ns::Descriptor* OptionsRegistry::GetProtobufDescriptor(
-    const std::string& type_name) {
-  const proto_ns::Descriptor* result =
-      proto_ns::DescriptorPool::generated_pool()->FindMessageTypeByName(
-          type_name);
-  if (!result) {
-    result = options_descriptor_pool()->FindMessageTypeByName(type_name);
+void OptionsRegistry::Register(const proto_ns::DescriptorProto& message_type,
+                               const std::string& parent_name) {
+  auto full_name = absl::StrCat(parent_name, ".", message_type.name());
+  descriptors()[full_name] = Descriptor(message_type, full_name);
+  for (auto& nested : message_type.nested_type()) {
+    Register(nested, full_name);
   }
-  return result;
+  for (auto& extension : message_type.extension()) {
+    extensions()[CanonicalTypeName(extension.extendee())].push_back(
+        FieldDescriptor(extension));
+  }
+}
+
+const Descriptor* OptionsRegistry::GetProtobufDescriptor(
+    const std::string& type_name) {
+  absl::ReaderMutexLock lock(&mutex());
+  auto it = descriptors().find(CanonicalTypeName(type_name));
+  return (it == descriptors().end()) ? nullptr : &it->second;
 }
 
 void OptionsRegistry::FindAllExtensions(
-    const proto_ns::Descriptor& extendee,
-    std::vector<const proto_ns::FieldDescriptor*>* result) {
-  using proto_ns::DescriptorPool;
-  std::vector<const proto_ns::FieldDescriptor*> extensions;
-  DescriptorPool::generated_pool()->FindAllExtensions(&extendee, &extensions);
-  options_descriptor_pool()->FindAllExtensions(&extendee, &extensions);
-  absl::flat_hash_set<int> numbers;
-  for (const proto_ns::FieldDescriptor* extension : extensions) {
-    bool inserted = numbers.insert(extension->number()).second;
-    if (inserted) {
-      result->push_back(extension);
+    absl::string_view extendee, std::vector<const FieldDescriptor*>* result) {
+  absl::ReaderMutexLock lock(&mutex());
+  result->clear();
+  if (extensions().count(extendee) > 0) {
+    for (const FieldDescriptor& field : extensions().at(extendee)) {
+      result->push_back(&field);
     }
   }
 }
 
+absl::flat_hash_map<std::string, Descriptor>& OptionsRegistry::descriptors() {
+  static auto* descriptors = new absl::flat_hash_map<std::string, Descriptor>();
+  return *descriptors;
+}
+
+absl::flat_hash_map<std::string, std::vector<FieldDescriptor>>&
+OptionsRegistry::extensions() {
+  static auto* extensions =
+      new absl::flat_hash_map<std::string, std::vector<FieldDescriptor>>();
+  return *extensions;
+}
+
+absl::Mutex& OptionsRegistry::mutex() {
+  static auto* mutex = new absl::Mutex();
+  return *mutex;
+}
+
+Descriptor::Descriptor(const proto_ns::DescriptorProto& proto,
+                       const std::string& full_name)
+    : full_name_(full_name) {
+  for (auto& field : proto.field()) {
+    fields_[field.name()] = FieldDescriptor(field);
+  }
+}
+
+const std::string& Descriptor::full_name() const { return full_name_; }
+
+const FieldDescriptor* Descriptor::FindFieldByName(
+    const std::string& name) const {
+  auto it = fields_.find(name);
+  return (it != fields_.end()) ? &it->second : nullptr;
+}
+
+FieldDescriptor::FieldDescriptor(const proto_ns::FieldDescriptorProto& proto) {
+  name_ = proto.name();
+  message_type_ = CanonicalTypeName(proto.type_name());
+  type_ = proto.type();
+  number_ = proto.number();
+}
+
+const std::string& FieldDescriptor::name() const { return name_; }
+
+int FieldDescriptor::number() const { return number_; }
+
+proto_ns::FieldDescriptorProto::Type FieldDescriptor::type() const {
+  return type_;
+}
+
+const Descriptor* FieldDescriptor::message_type() const {
+  return OptionsRegistry::GetProtobufDescriptor(message_type_);
+}
+
 }  // namespace tool
 }  // namespace mediapipe

mediapipe/framework/tool/options_registry.h

@@ -1,12 +1,16 @@
 #ifndef MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_REGISTRY_H_
 #define MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_REGISTRY_H_
 
+#include "absl/container/flat_hash_map.h"
 #include "mediapipe/framework/deps/registration.h"
 #include "mediapipe/framework/port/advanced_proto_inc.h"
 
 namespace mediapipe {
 namespace tool {
 
+class Descriptor;
+class FieldDescriptor;
+
 // A static registry that stores descriptors for protobufs used in MediaPipe
 // calculator options. Lite-proto builds do not normally include descriptors.
 // These registered descriptors allow individual protobuf fields to be
@@ -17,23 +21,60 @@ class OptionsRegistry {
   static RegistrationToken Register(const proto_ns::FileDescriptorSet& files);
 
   // Finds the descriptor for a protobuf.
-  static const proto_ns::Descriptor* GetProtobufDescriptor(
-      const std::string& type_name);
+  static const Descriptor* GetProtobufDescriptor(const std::string& type_name);
 
   // Returns all known proto2 extensions to a type.
-  static void FindAllExtensions(
-      const proto_ns::Descriptor& extendee,
-      std::vector<const proto_ns::FieldDescriptor*>* result);
+  static void FindAllExtensions(absl::string_view extendee,
+                                std::vector<const FieldDescriptor*>* result);
 
  private:
-  // Stores the descriptors for each options protobuf type.
-  static proto_ns::DescriptorPool* options_descriptor_pool();
+  // Registers protobuf descriptors a MessageLite and nested types.
+  static void Register(const proto_ns::DescriptorProto& message_type,
+                       const std::string& parent_name);
+
+  static absl::flat_hash_map<std::string, Descriptor>& descriptors();
+  static absl::flat_hash_map<std::string, std::vector<FieldDescriptor>>&
+  extensions();
+  static absl::Mutex& mutex();
 
   // Registers the descriptors for each options protobuf type.
   template <class MessageT>
   static const RegistrationToken registration_token;
 };
 
+// A custom implementation proto_ns::Descriptor.  This implementation
+// avoids a code size problem introduced by proto_ns::FieldDescriptor.
+class Descriptor {
+ public:
+  Descriptor() {}
+  Descriptor(const proto_ns::DescriptorProto& proto,
+             const std::string& full_name);
+  const std::string& full_name() const;
+  const FieldDescriptor* FindFieldByName(const std::string& name) const;
+
+ private:
+  std::string full_name_;
+  absl::flat_hash_map<std::string, FieldDescriptor> fields_;
+};
+
+// A custom implementation proto_ns::FieldDescriptor.  This implementation
+// avoids a code size problem introduced by proto_ns::FieldDescriptor.
+class FieldDescriptor {
+ public:
+  FieldDescriptor() {}
+  FieldDescriptor(const proto_ns::FieldDescriptorProto& proto);
+  const std::string& name() const;
+  int number() const;
+  proto_ns::FieldDescriptorProto::Type type() const;
+  const Descriptor* message_type() const;
+
+ private:
+  std::string name_;
+  std::string message_type_;
+  proto_ns::FieldDescriptorProto::Type type_;
+  int number_;
+};
+
 }  // namespace tool
 }  // namespace mediapipe
 

mediapipe/framework/tool/options_syntax_util.cc

@@ -0,0 +1,198 @@
+#include "mediapipe/framework/tool/options_syntax_util.h"
+
+#include <memory>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include "absl/strings/match.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/packet_type.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/any_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/name_util.h"
+#include "mediapipe/framework/tool/options_registry.h"
+
+namespace mediapipe {
+namespace tool {
+
+namespace {
+
+// StrSplit Delimiter to split strings at single colon tokens, ignoring
+// double-colon tokens.
+class SingleColonDelimiter {
+ public:
+  SingleColonDelimiter() {}
+  absl::string_view Find(absl::string_view text, size_t pos) const {
+    while (pos < text.length()) {
+      size_t p = text.find(':', pos);
+      p = (p == absl::string_view::npos) ? text.length() : p;
+      if (p >= text.length() - 1 || text[p + 1] != ':') {
+        return text.substr(p, 1);
+      }
+      pos = p + 2;
+    }
+    return text.substr(text.length(), 0);
+  }
+};
+
+}  // namespace
+
+// Helper functions for parsing the graph options syntax.
+class OptionsSyntaxUtil::OptionsSyntaxHelper {
+ public:
+  // The usual graph options syntax tokens.
+  OptionsSyntaxHelper() : syntax_{"OPTIONS", "options", "/"} {}
+
+  // Returns the tag name for an option protobuf field.
+  std::string OptionFieldTag(const std::string& name) { return name; }
+
+  // Returns the packet name for an option protobuf field.
+  absl::string_view OptionFieldPacket(absl::string_view name) { return name; }
+
+  // Returns the option protobuf field name for a tag or packet name.
+  absl::string_view OptionFieldName(absl::string_view name) { return name; }
+
+  // Return the extension-type specified for an option field.
+  absl::string_view ExtensionType(absl::string_view option_name) {
+    constexpr absl::string_view kExt = "Ext::";
+    if (absl::StartsWithIgnoreCase(option_name, kExt)) {
+      return option_name.substr(kExt.size());
+    }
+    return "";
+  }
+
+  // Returns the field names encoded in an options tag.
+  std::vector<absl::string_view> OptionTagNames(absl::string_view tag) {
+    if (absl::StartsWith(tag, syntax_.tag_name)) {
+      tag = tag.substr(syntax_.tag_name.length());
+    } else if (absl::StartsWith(tag, syntax_.packet_name)) {
+      tag = tag.substr(syntax_.packet_name.length());
+    }
+    if (absl::StartsWith(tag, syntax_.separator)) {
+      tag = tag.substr(syntax_.separator.length());
+    }
+    return absl::StrSplit(tag, syntax_.separator);
+  }
+
+  // Returns the field-path for an option stream-tag.
+  FieldPath OptionFieldPath(absl::string_view tag,
+                            const Descriptor* descriptor) {
+    std::vector<absl::string_view> name_tags = OptionTagNames(tag);
+    FieldPath result;
+    for (absl::string_view name_tag : name_tags) {
+      if (name_tag.empty()) {
+        continue;
+      }
+      absl::string_view option_name = OptionFieldName(name_tag);
+      int index;
+      if (absl::SimpleAtoi(option_name, &index)) {
+        result.back().index = index;
+      }
+      if (!ExtensionType(option_name).empty()) {
+        std::string extension_type = std::string(ExtensionType(option_name));
+        result.push_back({nullptr, 0, extension_type});
+        descriptor = OptionsRegistry::GetProtobufDescriptor(extension_type);
+      } else {
+        if (descriptor == nullptr) {
+          break;
+        }
+        auto field = descriptor->FindFieldByName(std::string(option_name));
+        descriptor = field ? field->message_type() : nullptr;
+        result.push_back({std::move(field), 0});
+      }
+    }
+    return result;
+  }
+
+  // Returns the option field name for a graph options packet name.
+  std::string GraphOptionFieldName(const std::string& graph_option_name) {
+    int prefix = syntax_.packet_name.length() + syntax_.separator.length();
+    std::string result = graph_option_name;
+    result.erase(0, prefix);
+    return result;
+  }
+
+  // Returns the graph options packet name for an option field name.
+  std::string GraphOptionName(const std::string& option_name) {
+    std::string packet_prefix =
+        syntax_.packet_name + absl::AsciiStrToLower(syntax_.separator);
+    return absl::StrCat(packet_prefix, option_name);
+  }
+
+  // Returns the tag name for a graph option.
+  std::string OptionTagName(const std::string& option_name) {
+    return absl::StrCat(syntax_.tag_name, syntax_.separator,
+                        OptionFieldTag(option_name));
+  }
+
+  // Converts slash-separated field names into a tag name.
+  std::string OptionFieldsTag(absl::string_view option_names) {
+    std::string tag_prefix = syntax_.tag_name + syntax_.separator;
+    std::vector<absl::string_view> names = absl::StrSplit(option_names, '/');
+    if (!names.empty() && names[0] == syntax_.tag_name) {
+      names.erase(names.begin());
+    }
+    if (!names.empty() && names[0] == syntax_.packet_name) {
+      names.erase(names.begin());
+    }
+    std::string result;
+    std::string sep = "";
+    for (absl::string_view v : names) {
+      absl::StrAppend(&result, sep, OptionFieldTag(std::string(v)));
+      sep = syntax_.separator;
+    }
+    result = tag_prefix + result;
+    return result;
+  }
+
+  // Token definitions for the graph options syntax.
+  struct OptionsSyntax {
+    // The tag name for an options protobuf.
+    std::string tag_name;
+    // The packet name for an options protobuf.
+    std::string packet_name;
+    // The separator between nested options fields.
+    std::string separator;
+  };
+
+  OptionsSyntax syntax_;
+};  // class OptionsSyntaxHelper
+
+OptionsSyntaxUtil::OptionsSyntaxUtil()
+    : syntax_helper_(std::make_unique<OptionsSyntaxHelper>()) {}
+
+OptionsSyntaxUtil::OptionsSyntaxUtil(const std::string& tag_name)
+    : OptionsSyntaxUtil() {
+  syntax_helper_->syntax_.tag_name = tag_name;
+}
+
+OptionsSyntaxUtil::OptionsSyntaxUtil(const std::string& tag_name,
+                                     const std::string& packet_name,
+                                     const std::string& separator)
+    : OptionsSyntaxUtil() {
+  syntax_helper_->syntax_.tag_name = tag_name;
+  syntax_helper_->syntax_.packet_name = packet_name;
+  syntax_helper_->syntax_.separator = separator;
+}
+
+OptionsSyntaxUtil::~OptionsSyntaxUtil() {}
+
+std::string OptionsSyntaxUtil::OptionFieldsTag(absl::string_view option_names) {
+  return syntax_helper_->OptionFieldsTag(option_names);
+}
+
+OptionsSyntaxUtil::FieldPath OptionsSyntaxUtil::OptionFieldPath(
+    absl::string_view tag, const Descriptor* descriptor) {
+  return syntax_helper_->OptionFieldPath(tag, descriptor);
+}
+std::vector<absl::string_view> OptionsSyntaxUtil::StrSplitTags(
+    absl::string_view tag_and_name) {
+  return absl::StrSplit(tag_and_name, SingleColonDelimiter());
+}
+
+}  // namespace tool
+}  // namespace mediapipe

mediapipe/framework/tool/options_syntax_util.h

@@ -0,0 +1,48 @@
+#ifndef MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_SYNTAX_UTIL_H_
+#define MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_SYNTAX_UTIL_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/packet_type.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/tool/options_field_util.h"
+#include "mediapipe/framework/tool/options_registry.h"
+
+namespace mediapipe {
+namespace tool {
+
+// Utility to parse the graph options syntax used in "option_value",
+// "side_packet", and "stream".
+class OptionsSyntaxUtil {
+ public:
+  using FieldPath = options_field_util::FieldPath;
+  OptionsSyntaxUtil();
+  OptionsSyntaxUtil(const std::string& tag_name);
+  OptionsSyntaxUtil(const std::string& tag_name, const std::string& packet_name,
+                    const std::string& separator);
+  ~OptionsSyntaxUtil();
+
+  // Converts slash-separated field names into a tag name.
+  std::string OptionFieldsTag(absl::string_view option_names);
+
+  // Returns the field-path for an option stream-tag.
+  FieldPath OptionFieldPath(absl::string_view tag,
+                            const Descriptor* descriptor);
+
+  // Splits a std::string into "tag" and "name" delimited by a single colon.
+  std::vector<absl::string_view> StrSplitTags(absl::string_view tag_and_name);
+
+ private:
+  class OptionsSyntaxHelper;
+  std::unique_ptr<OptionsSyntaxHelper> syntax_helper_;
+};
+
+}  // namespace tool
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_TOOL_OPTIONS_SYNTAX_UTIL_H_

mediapipe/framework/tool/options_util.cc

@@ -1,16 +1,110 @@
 
 #include "mediapipe/framework/tool/options_util.h"
 
-#include "mediapipe/framework/port/proto_ns.h"
+#include <memory>
+#include <string>
+#include <variant>
+
+#include "absl/strings/ascii.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/input_stream_shard.h"
+#include "mediapipe/framework/output_side_packet.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/packet_set.h"
+#include "mediapipe/framework/packet_type.h"
+#include "mediapipe/framework/port/advanced_proto_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/name_util.h"
+#include "mediapipe/framework/tool/options_field_util.h"
+#include "mediapipe/framework/tool/options_registry.h"
+#include "mediapipe/framework/tool/options_syntax_util.h"
+#include "mediapipe/framework/tool/proto_util_lite.h"
 
 namespace mediapipe {
 namespace tool {
 
-// TODO: Return registered protobuf Descriptors when available.
-const proto_ns::Descriptor* GetProtobufDescriptor(
-    const std::string& type_name) {
-  return proto_ns::DescriptorPool::generated_pool()->FindMessageTypeByName(
-      type_name);
+using options_field_util::FieldPath;
+using options_field_util::GetField;
+using options_field_util::GetGraphOptions;
+using options_field_util::GetNodeOptions;
+using options_field_util::MergeField;
+using options_field_util::MergeMessages;
+
+// Returns the type for the root options message if specified.
+std::string ExtensionType(const std::string& option_fields_tag) {
+  OptionsSyntaxUtil syntax_util;
+  options_field_util::FieldPath field_path =
+      syntax_util.OptionFieldPath(option_fields_tag, nullptr);
+  std::string result = !field_path.empty() ? field_path[0].extension_type : "";
+  return !result.empty() ? result : "*";
+}
+
+// Constructs a FieldPath for field names starting at a message type.
+FieldPath GetPath(const std::string& path_tag,
+                  const std::string& message_type) {
+  OptionsSyntaxUtil syntax_util;
+  const Descriptor* descriptor =
+      OptionsRegistry::GetProtobufDescriptor(message_type);
+  return syntax_util.OptionFieldPath(path_tag, descriptor);
+}
+
+// Returns the message type for a FieldData.
+std::string MessageType(FieldData message) {
+  return options_field_util::ParseTypeUrl(
+      std::string(message.message_value().type_url()));
+}
+
+// Copy literal options from graph_options to node_options.
+absl::Status CopyLiteralOptions(CalculatorGraphConfig::Node parent_node,
+                                CalculatorGraphConfig* config) {
+  Status status;
+  FieldData graph_data = options_field_util::AsFieldData(*config);
+  FieldData parent_data = options_field_util::AsFieldData(parent_node);
+
+  OptionsSyntaxUtil syntax_util;
+  for (auto& node : *config->mutable_node()) {
+    FieldData node_data = options_field_util::AsFieldData(node);
+
+    for (const std::string& option_def : node.option_value()) {
+      std::vector<absl::string_view> tag_and_name =
+          syntax_util.StrSplitTags(option_def);
+      std::string graph_tag = syntax_util.OptionFieldsTag(tag_and_name[1]);
+      std::string graph_extension_type = ExtensionType(graph_tag);
+      std::string node_tag = syntax_util.OptionFieldsTag(tag_and_name[0]);
+      std::string node_extension_type = ExtensionType(node_tag);
+      FieldData graph_options;
+      GetGraphOptions(graph_data, graph_extension_type, &graph_options)
+          .IgnoreError();
+      FieldData parent_options;
+      GetNodeOptions(parent_data, graph_extension_type, &parent_options)
+          .IgnoreError();
+      status.Update(
+          MergeMessages(graph_options, parent_options, &graph_options));
+      FieldData node_options;
+      status.Update(
+          GetNodeOptions(node_data, node_extension_type, &node_options));
+      if (!node_options.has_message_value() ||
+          !graph_options.has_message_value()) {
+        continue;
+      }
+      FieldPath graph_path = GetPath(graph_tag, MessageType(graph_options));
+      FieldPath node_path = GetPath(node_tag, MessageType(node_options));
+      FieldData packet_data;
+      status.Update(GetField(graph_path, graph_options, &packet_data));
+      status.Update(MergeField(node_path, packet_data, &node_options));
+      options_field_util::SetOptionsMessage(node_options, &node);
+    }
+  }
+  return status;
+}
+
+// Makes all configuration modifications needed for graph options.
+absl::Status DefineGraphOptions(const CalculatorGraphConfig::Node& parent_node,
+                                CalculatorGraphConfig* config) {
+  MP_RETURN_IF_ERROR(CopyLiteralOptions(parent_node, config));
+  return mediapipe::OkStatus();
 }
 
 }  // namespace tool

mediapipe/framework/tool/options_util.h

@@ -21,7 +21,6 @@
 #include "mediapipe/framework/packet_set.h"
 #include "mediapipe/framework/port/any_proto.h"
 #include "mediapipe/framework/tool/options_map.h"
-#include "mediapipe/framework/tool/type_util.h"
 
 namespace mediapipe {
 
@@ -75,8 +74,9 @@ inline T RetrieveOptions(const T& base, const InputStreamShardSet& stream_set,
   return base;
 }
 
-// Finds the descriptor for a protobuf.
-const proto_ns::Descriptor* GetProtobufDescriptor(const std::string& type_name);
+// Copy literal options from enclosing graphs.
+absl::Status DefineGraphOptions(const CalculatorGraphConfig::Node& parent_node,
+                                CalculatorGraphConfig* config);
 
 }  // namespace tool
 }  // namespace mediapipe

mediapipe/framework/tool/options_util_test.cc

@@ -15,16 +15,73 @@
 #include <memory>
 #include <vector>
 
+#include "absl/strings/string_view.h"
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/message_matchers.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/testdata/night_light_calculator.pb.h"
+#include "mediapipe/framework/tool/node_chain_subgraph.pb.h"
+#include "mediapipe/framework/tool/options_field_util.h"
 #include "mediapipe/framework/tool/options_registry.h"
+#include "mediapipe/framework/tool/options_syntax_util.h"
 
 namespace mediapipe {
 namespace {
 
+using ::mediapipe::proto_ns::FieldDescriptorProto;
+using FieldType = ::mediapipe::proto_ns::FieldDescriptorProto::Type;
+
+// A test Calculator using DeclareOptions and DefineOptions.
+class NightLightCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc) {
+    return mediapipe::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) final {
+    return mediapipe::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    return mediapipe::OkStatus();
+  }
+
+ private:
+  NightLightCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(NightLightCalculator);
+
+using tool::options_field_util::FieldPath;
+
+// Validates FieldPathEntry contents.
+bool Equals(const tool::options_field_util::FieldPathEntry& entry,
+            const std::string& field_name, int index,
+            const std::string& extension_type) {
+  const std::string& name = entry.field ? entry.field->name() : "";
+  return name == field_name && entry.index == index &&
+         entry.extension_type == extension_type;
+}
+
+// Serializes a MessageLite into FieldData.message_value.
+FieldData AsFieldData(const proto_ns::MessageLite& message) {
+  FieldData result;
+  *result.mutable_message_value()->mutable_value() =
+      message.SerializeAsString();
+  result.mutable_message_value()->set_type_url(message.GetTypeName());
+  return result;
+}
+
+// Returns the type for the root options message if specified.
+std::string ExtensionType(const std::string& option_fields_tag) {
+  tool::OptionsSyntaxUtil syntax_util;
+  tool::options_field_util::FieldPath field_path =
+      syntax_util.OptionFieldPath(option_fields_tag, nullptr);
+  std::string result = !field_path.empty() ? field_path[0].extension_type : "";
+  return !result.empty() ? result : "*";
+}
+
 // Tests for calculator and graph options.
 //
 class OptionsUtilTest : public ::testing::Test {
@@ -35,21 +92,198 @@ class OptionsUtilTest : public ::testing::Test {
 
 // Retrieves the description of a protobuf.
 TEST_F(OptionsUtilTest, GetProtobufDescriptor) {
-  const proto_ns::Descriptor* descriptor =
-      tool::GetProtobufDescriptor("mediapipe.CalculatorGraphConfig");
-#ifndef MEDIAPIPE_MOBILE
+  const tool::Descriptor* descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(
+          "mediapipe.CalculatorGraphConfig");
   EXPECT_NE(nullptr, descriptor);
-#else
-  EXPECT_EQ(nullptr, descriptor);
-#endif
 }
 
-// Retrieves the description of a protobuf from the OptionsRegistry.
+// Shows a calculator node deriving options from graph options.
+// The subgraph specifies "graph_options" as "NodeChainSubgraphOptions".
+// The calculator specifies "node_options as "NightLightCalculatorOptions".
+TEST_F(OptionsUtilTest, CopyLiteralOptions) {
+  CalculatorGraphConfig subgraph_config;
+
+  auto node = subgraph_config.add_node();
+  *node->mutable_calculator() = "NightLightCalculator";
+  *node->add_option_value() = "num_lights:options/chain_length";
+
+  // The options framework requires at least an empty options protobuf
+  // as an indication the options protobuf type expected by the node.
+  NightLightCalculatorOptions node_options;
+  node->add_node_options()->PackFrom(node_options);
+
+  NodeChainSubgraphOptions options;
+  options.set_chain_length(8);
+  subgraph_config.add_graph_options()->PackFrom(options);
+  subgraph_config.set_type("NightSubgraph");
+
+  CalculatorGraphConfig graph_config;
+  node = graph_config.add_node();
+  *node->mutable_calculator() = "NightSubgraph";
+
+  CalculatorGraph graph;
+  graph_config.set_num_threads(4);
+  MP_EXPECT_OK(graph.Initialize({subgraph_config, graph_config}, {}, {}));
+
+  CalculatorGraphConfig expanded_config = graph.Config();
+  expanded_config.clear_executor();
+  CalculatorGraphConfig::Node actual_node;
+  actual_node = expanded_config.node(0);
+
+  CalculatorGraphConfig::Node expected_node;
+  expected_node.set_name("nightsubgraph__NightLightCalculator");
+  expected_node.set_calculator("NightLightCalculator");
+  NightLightCalculatorOptions expected_node_options;
+  expected_node_options.add_num_lights(8);
+  expected_node.add_node_options()->PackFrom(expected_node_options);
+  *expected_node.add_option_value() = "num_lights:options/chain_length";
+  EXPECT_THAT(actual_node, EqualsProto(expected_node));
+
+  MP_EXPECT_OK(graph.StartRun({}));
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  // Ensure static protobuf packet registration.
+  MakePacket<NodeChainSubgraphOptions>();
+  MakePacket<NightLightCalculatorOptions>();
+}
+
+// Retrieves the description of a protobuf message and a nested protobuf message
+// from the OptionsRegistry.
 TEST_F(OptionsUtilTest, GetProtobufDescriptorRegistered) {
-  const proto_ns::Descriptor* descriptor =
+  const tool::Descriptor* options_descriptor =
       tool::OptionsRegistry::GetProtobufDescriptor(
           "mediapipe.NightLightCalculatorOptions");
-  EXPECT_NE(nullptr, descriptor);
+  EXPECT_NE(nullptr, options_descriptor);
+  const tool::Descriptor* bundle_descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(
+          "mediapipe.NightLightCalculatorOptions.LightBundle");
+  EXPECT_NE(nullptr, bundle_descriptor);
+  EXPECT_EQ(options_descriptor->full_name(),
+            "mediapipe.NightLightCalculatorOptions");
+  const tool::FieldDescriptor* bundle_field =
+      options_descriptor->FindFieldByName("bundle");
+  EXPECT_EQ(bundle_field->message_type(), bundle_descriptor);
+}
+
+// Constructs the FieldPath for a nested node-option.
+TEST_F(OptionsUtilTest, OptionsSyntaxUtil) {
+  const tool::Descriptor* descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(
+          "mediapipe.NightLightCalculatorOptions");
+  std::string tag;
+  tool::OptionsSyntaxUtil::FieldPath field_path;
+  {
+    // The default tag syntax.
+    tool::OptionsSyntaxUtil syntax_util;
+    tag = syntax_util.OptionFieldsTag("options/sub_options/num_lights");
+    EXPECT_EQ(tag, "OPTIONS/sub_options/num_lights");
+    field_path = syntax_util.OptionFieldPath(tag, descriptor);
+    EXPECT_EQ(field_path.size(), 2);
+    EXPECT_EQ(field_path[0].field->name(), "sub_options");
+    EXPECT_EQ(field_path[1].field->name(), "num_lights");
+  }
+  {
+    // A tag syntax with a text-coded separator.
+    tool::OptionsSyntaxUtil syntax_util("OPTIONS", "options", "_Z0Z_");
+    tag = syntax_util.OptionFieldsTag("options/sub_options/num_lights");
+    EXPECT_EQ(tag, "OPTIONS_Z0Z_sub_options_Z0Z_num_lights");
+    field_path = syntax_util.OptionFieldPath(tag, descriptor);
+    EXPECT_EQ(field_path.size(), 2);
+    EXPECT_EQ(field_path[0].field->name(), "sub_options");
+    EXPECT_EQ(field_path[1].field->name(), "num_lights");
+  }
+}
+
+TEST_F(OptionsUtilTest, OptionFieldPath) {
+  tool::OptionsSyntaxUtil syntax_util;
+  std::vector<absl::string_view> split;
+  split = syntax_util.StrSplitTags("a/graph/option:a/node/option");
+  EXPECT_EQ(2, split.size());
+  EXPECT_EQ(split[0], "a/graph/option");
+  EXPECT_EQ(split[1], "a/node/option");
+  split = syntax_util.StrSplitTags("Ext::a/graph/option:Ext::a/node/option");
+  EXPECT_EQ(2, split.size());
+  EXPECT_EQ(split[0], "Ext::a/graph/option");
+  EXPECT_EQ(split[1], "Ext::a/node/option");
+
+  split =
+      syntax_util.StrSplitTags("chain_length:options/sub_options/num_lights");
+  EXPECT_EQ(2, split.size());
+  EXPECT_EQ(split[0], "chain_length");
+  EXPECT_EQ(split[1], "options/sub_options/num_lights");
+  const tool::Descriptor* descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(
+          "mediapipe.NightLightCalculatorOptions");
+  tool::options_field_util::FieldPath field_path =
+      syntax_util.OptionFieldPath(split[1], descriptor);
+  EXPECT_EQ(field_path.size(), 2);
+  EXPECT_EQ(field_path[0].field->name(), "sub_options");
+  EXPECT_EQ(field_path[1].field->name(), "num_lights");
+}
+
+TEST_F(OptionsUtilTest, FindOptionsMessage) {
+  tool::OptionsSyntaxUtil syntax_util;
+  std::vector<absl::string_view> split;
+  split =
+      syntax_util.StrSplitTags("chain_length:options/sub_options/num_lights");
+  EXPECT_EQ(2, split.size());
+  EXPECT_EQ(split[0], "chain_length");
+  EXPECT_EQ(split[1], "options/sub_options/num_lights");
+  const tool::Descriptor* descriptor =
+      tool::OptionsRegistry::GetProtobufDescriptor(
+          "mediapipe.NightLightCalculatorOptions");
+  tool::options_field_util::FieldPath field_path =
+      syntax_util.OptionFieldPath(split[1], descriptor);
+  EXPECT_EQ(field_path.size(), 2);
+  EXPECT_TRUE(Equals(field_path[0], "sub_options", 0, ""));
+  EXPECT_TRUE(Equals(field_path[1], "num_lights", 0, ""));
+
+  {
+    // NightLightCalculatorOptions in Node.options.
+    CalculatorGraphConfig::Node node;
+    NightLightCalculatorOptions* options =
+        node.mutable_options()->MutableExtension(
+            NightLightCalculatorOptions::ext);
+    options->mutable_sub_options()->add_num_lights(33);
+
+    // Retrieve the specified option.
+    FieldData node_data = AsFieldData(node);
+    auto path = field_path;
+    std::string node_extension_type = ExtensionType(std::string(split[1]));
+    FieldData node_options;
+    MP_EXPECT_OK(tool::options_field_util::GetNodeOptions(
+        node_data, node_extension_type, &node_options));
+    FieldData packet_data;
+    MP_EXPECT_OK(tool::options_field_util::GetField(field_path, node_options,
+                                                    &packet_data));
+    EXPECT_EQ(packet_data.value_case(), FieldData::kInt32Value);
+    EXPECT_EQ(packet_data.int32_value(), 33);
+  }
+
+  {
+    // NightLightCalculatorOptions in Node.node_options.
+    CalculatorGraphConfig::Node node;
+    NightLightCalculatorOptions options;
+    options.mutable_sub_options()->add_num_lights(33);
+    node.add_node_options()->PackFrom(options);
+
+    // Retrieve the specified option.
+    FieldData node_data = AsFieldData(node);
+    auto path = field_path;
+    std::string node_extension_type = ExtensionType(std::string(split[1]));
+    FieldData node_options;
+    MP_EXPECT_OK(tool::options_field_util::GetNodeOptions(
+        node_data, node_extension_type, &node_options));
+    FieldData packet_data;
+    MP_EXPECT_OK(tool::options_field_util::GetField(field_path, node_options,
+                                                    &packet_data));
+    EXPECT_EQ(packet_data.value_case(), FieldData::kInt32Value);
+    EXPECT_EQ(packet_data.int32_value(), 33);
+  }
+
+  // TODO: Test with specified extension_type.
 }
 
 }  // namespace

mediapipe/framework/tool/proto_util_lite.cc

@@ -196,6 +196,27 @@ absl::Status ProtoUtilLite::GetFieldRange(
   return absl::OkStatus();
 }
 
+// Returns the number of field values in a repeated protobuf field.
+absl::Status ProtoUtilLite::GetFieldCount(const FieldValue& message,
+                                          ProtoPath proto_path,
+                                          FieldType field_type,
+                                          int* field_count) {
+  int field_id, index;
+  std::tie(field_id, index) = proto_path.back();
+  proto_path.pop_back();
+  std::vector<std::string> parent;
+  if (proto_path.empty()) {
+    parent.push_back(std::string(message));
+  } else {
+    MP_RETURN_IF_ERROR(ProtoUtilLite::GetFieldRange(
+        message, proto_path, 1, WireFormatLite::TYPE_MESSAGE, &parent));
+  }
+  FieldAccess access(field_id, field_type);
+  MP_RETURN_IF_ERROR(access.SetMessage(parent[0]));
+  *field_count = access.mutable_field_values()->size();
+  return absl::OkStatus();
+}
+
 // If ok, returns OkStatus, otherwise returns InvalidArgumentError.
 template <typename T>
 absl::Status SyntaxStatus(bool ok, const std::string& text, T* result) {

mediapipe/framework/tool/proto_util_lite.h

@@ -75,6 +75,11 @@ class ProtoUtilLite {
                                     FieldType field_type,
                                     std::vector<FieldValue>* field_values);
 
+  // Returns the number of field values in a repeated protobuf field.
+  static absl::Status GetFieldCount(const FieldValue& message,
+                                    ProtoPath proto_path, FieldType field_type,
+                                    int* field_count);
+
   // Serialize one or more protobuf field values from text.
   static absl::Status Serialize(const std::vector<std::string>& text_values,
                                 FieldType field_type,