Project import generated by Copybara.

GitOrigin-RevId: 373e3ac1e5839befd95bf7d73ceff3c5f1171969

docs/getting_started/android_solutions.md

@@ -26,15 +26,17 @@ the following into the project's Gradle dependencies:
 
 ```
 dependencies {
-    // MediaPipe solution-core is the foundation of any MediaPipe solutions.
+    // MediaPipe solution-core is the foundation of any MediaPipe Solutions.
     implementation 'com.google.mediapipe:solution-core:latest.release'
-    // Optional: MediaPipe Hands solution.
-    implementation 'com.google.mediapipe:hands:latest.release'
-    // Optional: MediaPipe FaceMesh solution.
+    // Optional: MediaPipe Face Detection Solution.
+    implementation 'com.google.mediapipe:facedetection:latest.release'
+    // Optional: MediaPipe Face Mesh Solution.
     implementation 'com.google.mediapipe:facemesh:latest.release'
+    // Optional: MediaPipe Hands Solution.
+    implementation 'com.google.mediapipe:hands:latest.release'
     // MediaPipe deps
-    implementation 'com.google.flogger:flogger:latest.release'
-    implementation 'com.google.flogger:flogger-system-backend:latest.release'
+    implementation 'com.google.flogger:flogger:0.6'
+    implementation 'com.google.flogger:flogger-system-backend:0.6'
     implementation 'com.google.guava:guava:27.0.1-android'
     implementation 'com.google.protobuf:protobuf-java:3.11.4'
     // CameraX core library
@@ -45,7 +47,7 @@ dependencies {
 }
 ```
 
-See the detailed solutions API usage examples for different use cases in the
+See the detailed solution APIs usage examples for different use cases in the
 solution example apps'
 [source code](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions).
 If the prebuilt maven packages are not sufficient, building the MediaPipe

docs/getting_started/faq.md

@@ -103,7 +103,7 @@ monotonically increasing timestamps. By convention, realtime calculators and
 graphs use the recording time or the presentation time as the timestamp for each
 packet, with each timestamp representing microseconds since
 `Jan/1/1970:00:00:00`. This allows packets from various sources to be processed
-in a gloablly consistent order.
+in a globally consistent order.
 
 Normally for offline processing, every input packet is processed and processing
 continues as long as necessary. For online processing, it is often necessary to

docs/solutions/face_detection.md

@@ -121,12 +121,10 @@ with mp_face_detection.FaceDetection(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_detection.process(image)
 
     # Draw the face detection annotations on the image.
@@ -135,7 +133,8 @@ with mp_face_detection.FaceDetection(
     if results.detections:
       for detection in results.detections:
         mp_drawing.draw_detection(image, detection)
-    cv2.imshow('MediaPipe Face Detection', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Detection', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -200,7 +199,7 @@ const faceDetection = new FaceDetection({locateFile: (file) => {
   return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
 }});
 faceDetection.setOptions({
-  modelSelection: 0
+  modelSelection: 0,
   minDetectionConfidence: 0.5
 });
 faceDetection.onResults(onResults);
@@ -216,6 +215,194 @@ camera.start();
 </script>
 ```
 
+### Android Solution API
+
+Please first follow general
+[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
+to add MediaPipe Gradle dependencies, then try the Face Detection Solution API
+in the companion
+[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection)
+following
+[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio)
+and learn more in the usage example below.
+
+*   [staticImageMode](#static_image_mode)
+*   [modelSelection](#model_selection)
+
+#### Camera Input
+
+```java
+// For camera input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Detection Solution.
+CameraInput cameraInput = new CameraInput(this);
+cameraInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      RelativeKeypoint noseTip =
+          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+// The runnable to start camera after the GLSurfaceView is attached.
+glSurfaceView.post(
+    () ->
+        cameraInput.start(
+            this,
+            faceDetection.getGlContext(),
+            CameraInput.CameraFacing.FRONT,
+            glSurfaceView.getWidth(),
+            glSurfaceView.getHeight()));
+```
+
+#### Image Input
+
+```java
+// For reading images from gallery and drawing the output in an ImageView.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(true)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+
+// Connects MediaPipe Face Detection Solution to the user-defined ImageView
+// instance that allows users to have the custom drawing of the output landmarks
+// on it. See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultImageView.java
+// as an example.
+FaceDetectionResultImageView imageView = new FaceDetectionResultImageView(this);
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      int width = faceDetectionResult.inputBitmap().getWidth();
+      int height = faceDetectionResult.inputBitmap().getHeight();
+      RelativeKeypoint noseTip =
+          FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
+              noseTip.getX() * width, noseTip.getY() * height));
+      // Request canvas drawing.
+      imageView.setFaceDetectionResult(faceDetectionResult);
+      runOnUiThread(() -> imageView.update());
+    });
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// ActivityResultLauncher to get an image from the gallery as Bitmap.
+ActivityResultLauncher<Intent> imageGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null && result.getResultCode() == RESULT_OK) {
+            Bitmap bitmap = null;
+            try {
+              bitmap =
+                  MediaStore.Images.Media.getBitmap(
+                      this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
+            } catch (IOException e) {
+              Log.e(TAG, "Bitmap reading error:" + e);
+            }
+            if (bitmap != null) {
+              faceDetection.send(bitmap);
+            }
+          }
+        });
+Intent gallery = new Intent(
+    Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI);
+imageGetter.launch(gallery);
+```
+
+#### Video Input
+
+```java
+// For video input and result rendering with OpenGL.
+FaceDetectionOptions faceDetectionOptions =
+    FaceDetectionOptions.builder()
+        .setStaticImageMode(false)
+        .setModelSelection(0).build();
+FaceDetection faceDetection = new FaceDetection(this, faceDetectionOptions);
+faceDetection.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Detection Solution.
+VideoInput videoInput = new VideoInput(this);
+videoInput.setNewFrameListener(
+    textureFrame -> faceDetection.send(textureFrame));
+
+// Initializes a new GlSurfaceView with a ResultGlRenderer<FaceDetectionResult> instance
+// that provides the interfaces to run user-defined OpenGL rendering code.
+// See mediapipe/examples/android/solutions/facedetection/src/main/java/com/google/mediapipe/examples/facedetection/FaceDetectionResultGlRenderer.java
+// as an example.
+SolutionGlSurfaceView<FaceDetectionResult> glSurfaceView =
+    new SolutionGlSurfaceView<>(
+        this, faceDetection.getGlContext(), faceDetection.getGlMajorVersion());
+glSurfaceView.setSolutionResultRenderer(new FaceDetectionResultGlRenderer());
+glSurfaceView.setRenderInputImage(true);
+
+faceDetection.setResultListener(
+    faceDetectionResult -> {
+      RelativeKeypoint noseTip =
+        FaceDetection.getFaceKeypoint(result, 0, FaceKeypoint.NOSE_TIP);
+      Log.i(
+          TAG,
+          String.format(
+              "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              noseTip.getX(), noseTip.getY()));
+      // Request GL rendering.
+      glSurfaceView.setRenderData(faceDetectionResult);
+      glSurfaceView.requestRender();
+    });
+
+ActivityResultLauncher<Intent> videoGetter =
+    registerForActivityResult(
+        new ActivityResultContracts.StartActivityForResult(),
+        result -> {
+          Intent resultIntent = result.getData();
+          if (resultIntent != null) {
+            if (result.getResultCode() == RESULT_OK) {
+              glSurfaceView.post(
+                  () ->
+                      videoInput.start(
+                          this,
+                          resultIntent.getData(),
+                          faceDetection.getGlContext(),
+                          glSurfaceView.getWidth(),
+                          glSurfaceView.getHeight()));
+            }
+          }
+        });
+Intent gallery =
+    new Intent(Intent.ACTION_PICK, MediaStore.Video.Media.INTERNAL_CONTENT_URI);
+videoGetter.launch(gallery);
+```
+
 ## Example Apps
 
 Please first see general instructions for

docs/solutions/face_mesh.md

@@ -111,6 +111,23 @@ You can find more information about the face landmark model in this
 :------------------------------------------------------------------------: |
 *Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
 
+#### Attention Mesh Model
+
+In addition to the [Face Landmark Model](#face-landmark-model) we provide
+another model that applies
+[attention](https://en.wikipedia.org/wiki/Attention_(machine_learning)) to
+semantically meaningful face regions, and therefore predicting landmarks more
+accurately around lips, eyes and irises, at the expense of more compute. It
+enables applications like AR makeup and AR puppeteering.
+
+The attention mesh model can be selected in the Solution APIs via the
+[refine_landmarks](#refine_landmarks) option. You can also find more information
+about the model in this [paper](https://arxiv.org/abs/2006.10962).
+
+![attention_mesh_architecture.png](../images/attention_mesh_architecture.png) |
+:---------------------------------------------------------------------------: |
+*Fig 3. Attention Mesh: Overview of model architecture.*                      |
+
 ## Face Geometry Module
 
 The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
@@ -145,8 +162,8 @@ be set freely, however for better results it is advised to set them as close to
 the *real physical camera parameters* as possible.
 
 ![face_geometry_metric_3d_space.gif](../images/face_geometry_metric_3d_space.gif) |
-:----------------------------------------------------------------------------: |
-*Fig 3. A visualization of multiple key elements in the Metric 3D space.*      |
+:-------------------------------------------------------------------------------: |
+*Fig 4. A visualization of multiple key elements in the Metric 3D space.*         |
 
 #### Canonical Face Model
 
@@ -210,7 +227,7 @@ The effect renderer is implemented as a MediaPipe
 
 | ![face_geometry_renderer.gif](../images/face_geometry_renderer.gif)     |
 | :---------------------------------------------------------------------: |
-| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
+| *Fig 5. An example of face effects rendered by the Face Geometry Effect Renderer.* |
 
 ## Solution APIs
 
@@ -234,6 +251,12 @@ unrelated, images. Default to `false`.
 
 Maximum number of faces to detect. Default to `1`.
 
+#### refine_landmarks
+
+Whether to further refine the landmark coordinates around the eyes and lips, and
+output additional landmarks around the irises by applying the
+[Attention Mesh Model](#attention-mesh-model). Default to `false`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the face detection model for the
@@ -271,6 +294,7 @@ Supported configuration options:
 
 *   [static_image_mode](#static_image_mode)
 *   [max_num_faces](#max_num_faces)
+*   [refine_landmarks](#refine_landmarks)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -287,6 +311,7 @@ drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 with mp_face_mesh.FaceMesh(
     static_image_mode=True,
     max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5) as face_mesh:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
@@ -313,12 +338,21 @@ with mp_face_mesh.FaceMesh(
           landmark_drawing_spec=None,
           connection_drawing_spec=mp_drawing_styles
           .get_default_face_mesh_contours_style())
+      mp_drawing.draw_landmarks(
+          image=annotated_image,
+          landmark_list=face_landmarks,
+          connections=mp_face_mesh.FACEMESH_IRISES,
+          landmark_drawing_spec=None,
+          connection_drawing_spec=mp_drawing_styles
+          .get_default_face_mesh_iris_connections_style())
     cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image)
 
 # For webcam input:
 drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
 cap = cv2.VideoCapture(0)
 with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=True,
     min_detection_confidence=0.5,
     min_tracking_confidence=0.5) as face_mesh:
   while cap.isOpened():
@@ -328,12 +362,10 @@ with mp_face_mesh.FaceMesh(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = face_mesh.process(image)
 
     # Draw the face mesh annotations on the image.
@@ -355,7 +387,15 @@ with mp_face_mesh.FaceMesh(
             landmark_drawing_spec=None,
             connection_drawing_spec=mp_drawing_styles
             .get_default_face_mesh_contours_style())
-    cv2.imshow('MediaPipe FaceMesh', image)
+        mp_drawing.draw_landmarks(
+            image=image,
+            landmark_list=face_landmarks,
+            connections=mp_face_mesh.FACEMESH_IRISES,
+            landmark_drawing_spec=None,
+            connection_drawing_spec=mp_drawing_styles
+            .get_default_face_mesh_iris_connections_style())
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Face Mesh', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -370,6 +410,7 @@ and the following usage example.
 Supported configuration options:
 
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -410,8 +451,10 @@ function onResults(results) {
                      {color: '#C0C0C070', lineWidth: 1});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_RIGHT_IRIS, {color: '#FF3030'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
+      drawConnectors(canvasCtx, landmarks, FACEMESH_LEFT_IRIS, {color: '#30FF30'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
       drawConnectors(canvasCtx, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
     }
@@ -424,6 +467,7 @@ const faceMesh = new FaceMesh({locateFile: (file) => {
 }});
 faceMesh.setOptions({
   maxNumFaces: 1,
+  refineLandmarks: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });
@@ -444,7 +488,7 @@ camera.start();
 
 Please first follow general
 [instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the FaceMash solution API in the
+to add MediaPipe Gradle dependencies, then try the Face Mesh Solution API in the
 companion
 [example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh)
 following
@@ -455,6 +499,7 @@ Supported configuration options:
 
 *   [staticImageMode](#static_image_mode)
 *   [maxNumFaces](#max_num_faces)
+*   [refineLandmarks](#refine_landmarks)
 *   runOnGpu: Run the pipeline and the model inference on GPU or CPU.
 
 #### Camera Input
@@ -463,17 +508,18 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new CameraInput instance and connects it to MediaPipe Face Mesh Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -481,18 +527,18 @@ cameraInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -504,7 +550,7 @@ glSurfaceView.post(
     () ->
         cameraInput.start(
             this,
-            facemesh.getGlContext(),
+            faceMesh.getGlContext(),
             CameraInput.CameraFacing.FRONT,
             glSurfaceView.getWidth(),
             glSurfaceView.getHeight()));
@@ -516,17 +562,18 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumFaces(1)                           // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
 
-// Connects MediaPipe FaceMesh to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Face Mesh Solution to the user-defined ImageView instance
+// that allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/facemesh/src/main/java/com/google/mediapipe/examples/facemesh/FaceMeshResultImageView.java
 // as an example.
 FaceMeshResultImageView imageView = new FaceMeshResultImageView(this);
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       int width = faceMeshResult.inputBitmap().getWidth();
       int height = faceMeshResult.inputBitmap().getHeight();
@@ -535,14 +582,14 @@ facemesh.setResultListener(
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose coordinates (pixel values): x=%f, y=%f",
+              "MediaPipe Face Mesh nose coordinates (pixel values): x=%f, y=%f",
               noseLandmark.getX() * width, noseLandmark.getY() * height));
       // Request canvas drawing.
       imageView.setFaceMeshResult(faceMeshResult);
       runOnUiThread(() -> imageView.update());
     });
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
 // ActivityResultLauncher to get an image from the gallery as Bitmap.
 ActivityResultLauncher<Intent> imageGetter =
@@ -556,11 +603,12 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
             if (bitmap != null) {
-              facemesh.send(bitmap);
+              faceMesh.send(bitmap);
             }
           }
         });
@@ -575,17 +623,18 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 FaceMeshOptions faceMeshOptions =
     FaceMeshOptions.builder()
-        .setMode(FaceMeshOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumFaces(1)                        // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setRefineLandmarks(true)
+        .setMaxNumFaces(1)
         .setRunOnGpu(true).build();
-FaceMesh facemesh = new FaceMesh(this, faceMeshOptions);
-facemesh.setErrorListener(
-    (message, e) -> Log.e(TAG, "MediaPipe FaceMesh error:" + message));
+FaceMesh faceMesh = new FaceMesh(this, faceMeshOptions);
+faceMesh.setErrorListener(
+    (message, e) -> Log.e(TAG, "MediaPipe Face Mesh error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe FaceMesh.
+// Initializes a new VideoInput instance and connects it to MediaPipe Face Mesh Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
-    textureFrame -> facemesh.send(textureFrame));
+    textureFrame -> faceMesh.send(textureFrame));
 
 // Initializes a new GlSurfaceView with a ResultGlRenderer<FaceMeshResult> instance
 // that provides the interfaces to run user-defined OpenGL rendering code.
@@ -593,18 +642,18 @@ videoInput.setNewFrameListener(
 // as an example.
 SolutionGlSurfaceView<FaceMeshResult> glSurfaceView =
     new SolutionGlSurfaceView<>(
-        this, facemesh.getGlContext(), facemesh.getGlMajorVersion());
+        this, faceMesh.getGlContext(), faceMesh.getGlMajorVersion());
 glSurfaceView.setSolutionResultRenderer(new FaceMeshResultGlRenderer());
 glSurfaceView.setRenderInputImage(true);
 
-facemesh.setResultListener(
+faceMesh.setResultListener(
     faceMeshResult -> {
       NormalizedLandmark noseLandmark =
           result.multiFaceLandmarks().get(0).getLandmarkList().get(1);
       Log.i(
           TAG,
           String.format(
-              "MediaPipe FaceMesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
+              "MediaPipe Face Mesh nose normalized coordinates (value range: [0, 1]): x=%f, y=%f",
               noseLandmark.getX(), noseLandmark.getY()));
       // Request GL rendering.
       glSurfaceView.setRenderData(faceMeshResult);
@@ -623,7 +672,7 @@ ActivityResultLauncher<Intent> videoGetter =
                       videoInput.start(
                           this,
                           resultIntent.getData(),
-                          facemesh.getGlContext(),
+                          faceMesh.getGlContext(),
                           glSurfaceView.getWidth(),
                           glSurfaceView.getHeight()));
             }

docs/solutions/hands.md

@@ -269,12 +269,10 @@ with mp_hands.Hands(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = hands.process(image)
 
     # Draw the hand annotations on the image.
@@ -288,7 +286,8 @@ with mp_hands.Hands(
             mp_hands.HAND_CONNECTIONS,
             mp_drawing_styles.get_default_hand_landmarks_style(),
             mp_drawing_styles.get_default_hand_connections_style())
-    cv2.imshow('MediaPipe Hands', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Hands', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -372,7 +371,7 @@ camera.start();
 
 Please first follow general
 [instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api)
-to add MediaPipe Gradle dependencies, then try the Hands solution API in the
+to add MediaPipe Gradle dependencies, then try the Hands Solution API in the
 companion
 [example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands)
 following
@@ -391,14 +390,14 @@ Supported configuration options:
 // For camera input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new CameraInput instance and connects it to MediaPipe Hands.
+// Initializes a new CameraInput instance and connects it to MediaPipe Hands Solution.
 CameraInput cameraInput = new CameraInput(this);
 cameraInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));
@@ -444,13 +443,13 @@ glSurfaceView.post(
 // For reading images from gallery and drawing the output in an ImageView.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STATIC_IMAGE_MODE)  // API soon to become
-        .setMaxNumHands(1)                        // setStaticImageMode(true)
+        .setStaticImageMode(true)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 
-// Connects MediaPipe Hands to the user-defined ImageView instance that allows
-// users to have the custom drawing of the output landmarks on it.
+// Connects MediaPipe Hands Solution to the user-defined ImageView instance that
+// allows users to have the custom drawing of the output landmarks on it.
 // See mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java
 // as an example.
 HandsResultImageView imageView = new HandsResultImageView(this);
@@ -484,6 +483,7 @@ ActivityResultLauncher<Intent> imageGetter =
               bitmap =
                   MediaStore.Images.Media.getBitmap(
                       this.getContentResolver(), resultIntent.getData());
+              // Please also rotate the Bitmap based on its orientation.
             } catch (IOException e) {
               Log.e(TAG, "Bitmap reading error:" + e);
             }
@@ -503,14 +503,14 @@ imageGetter.launch(gallery);
 // For video input and result rendering with OpenGL.
 HandsOptions handsOptions =
     HandsOptions.builder()
-        .setMode(HandsOptions.STREAMING_MODE)  // API soon to become
-        .setMaxNumHands(1)                     // setStaticImageMode(false)
+        .setStaticImageMode(false)
+        .setMaxNumHands(1)
         .setRunOnGpu(true).build();
 Hands hands = new Hands(this, handsOptions);
 hands.setErrorListener(
     (message, e) -> Log.e(TAG, "MediaPipe Hands error:" + message));
 
-// Initializes a new VideoInput instance and connects it to MediaPipe Hands.
+// Initializes a new VideoInput instance and connects it to MediaPipe Hands Solution.
 VideoInput videoInput = new VideoInput(this);
 videoInput.setNewFrameListener(
     textureFrame -> hands.send(textureFrame));

docs/solutions/holistic.md

@@ -147,6 +147,18 @@ If set to `true`, the solution filters pose landmarks across different input
 images to reduce jitter, but ignored if [static_image_mode](#static_image_mode)
 is also set to `true`. Default to `true`.
 
+#### enable_segmentation
+
+If set to `true`, in addition to the pose, face and hand landmarks the solution
+also generates the segmentation mask. Default to `false`.
+
+#### smooth_segmentation
+
+If set to `true`, the solution filters segmentation masks across different input
+images to reduce jitter. Ignored if [enable_segmentation](#enable_segmentation)
+is `false` or [static_image_mode](#static_image_mode) is `true`. Default to
+`true`.
+
 #### min_detection_confidence
 
 Minimum confidence value (`[0.0, 1.0]`) from the person-detection model for the
@@ -207,6 +219,15 @@ the camera. The magnitude of `z` uses roughly the same scale as `x`.
 A list of 21 hand landmarks on the right hand, in the same representation as
 [left_hand_landmarks](#left_hand_landmarks).
 
+#### segmentation_mask
+
+The output segmentation mask, predicted only when
+[enable_segmentation](#enable_segmentation) is set to `true`. The mask has the
+same width and height as the input image, and contains values in `[0.0, 1.0]`
+where `1.0` and `0.0` indicate high certainty of a "human" and "background"
+pixel respectively. Please refer to the platform-specific usage examples below
+for usage details.
+
 ### Python Solution API
 
 Please first follow general [instructions](../getting_started/python.md) to
@@ -218,6 +239,8 @@ Supported configuration options:
 *   [static_image_mode](#static_image_mode)
 *   [model_complexity](#model_complexity)
 *   [smooth_landmarks](#smooth_landmarks)
+*   [enable_segmentation](#enable_segmentation)
+*   [smooth_segmentation](#smooth_segmentation)
 *   [min_detection_confidence](#min_detection_confidence)
 *   [min_tracking_confidence](#min_tracking_confidence)
 
@@ -232,7 +255,8 @@ mp_holistic = mp.solutions.holistic
 IMAGE_FILES = []
 with mp_holistic.Holistic(
     static_image_mode=True,
-    model_complexity=2) as holistic:
+    model_complexity=2,
+    enable_segmentation=True) as holistic:
   for idx, file in enumerate(IMAGE_FILES):
     image = cv2.imread(file)
     image_height, image_width, _ = image.shape
@@ -245,8 +269,16 @@ with mp_holistic.Holistic(
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].x * image_width}, '
           f'{results.pose_landmarks.landmark[mp_holistic.PoseLandmark.NOSE].y * image_height})'
       )
-    # Draw pose, left and right hands, and face landmarks on the image.
+
     annotated_image = image.copy()
+    # Draw segmentation on the image.
+    # To improve segmentation around boundaries, consider applying a joint
+    # bilateral filter to "results.segmentation_mask" with "image".
+    condition = np.stack((results.segmentation_mask,) * 3, axis=-1) > 0.1
+    bg_image = np.zeros(image.shape, dtype=np.uint8)
+    bg_image[:] = BG_COLOR
+    annotated_image = np.where(condition, annotated_image, bg_image)
+    # Draw pose, left and right hands, and face landmarks on the image.
     mp_drawing.draw_landmarks(
         annotated_image,
         results.face_landmarks,
@@ -277,12 +309,10 @@ with mp_holistic.Holistic(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = holistic.process(image)
 
     # Draw landmark annotation on the image.
@@ -301,7 +331,8 @@ with mp_holistic.Holistic(
         mp_holistic.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles
         .get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Holistic', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Holistic', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()
@@ -317,6 +348,8 @@ Supported configuration options:
 
 *   [modelComplexity](#model_complexity)
 *   [smoothLandmarks](#smooth_landmarks)
+*   [enableSegmentation](#enable_segmentation)
+*   [smoothSegmentation](#smooth_segmentation)
 *   [minDetectionConfidence](#min_detection_confidence)
 *   [minTrackingConfidence](#min_tracking_confidence)
 
@@ -349,8 +382,20 @@ const canvasCtx = canvasElement.getContext('2d');
 function onResults(results) {
   canvasCtx.save();
   canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
+  canvasCtx.drawImage(results.segmentationMask, 0, 0,
+                      canvasElement.width, canvasElement.height);
+
+  // Only overwrite existing pixels.
+  canvasCtx.globalCompositeOperation = 'source-in';
+  canvasCtx.fillStyle = '#00FF00';
+  canvasCtx.fillRect(0, 0, canvasElement.width, canvasElement.height);
+
+  // Only overwrite missing pixels.
+  canvasCtx.globalCompositeOperation = 'destination-atop';
   canvasCtx.drawImage(
       results.image, 0, 0, canvasElement.width, canvasElement.height);
+
+  canvasCtx.globalCompositeOperation = 'source-over';
   drawConnectors(canvasCtx, results.poseLandmarks, POSE_CONNECTIONS,
                  {color: '#00FF00', lineWidth: 4});
   drawLandmarks(canvasCtx, results.poseLandmarks,
@@ -374,6 +419,8 @@ const holistic = new Holistic({locateFile: (file) => {
 holistic.setOptions({
   modelComplexity: 1,
   smoothLandmarks: true,
+  enableSegmentation: true,
+  smoothSegmentation: true,
   minDetectionConfidence: 0.5,
   minTrackingConfidence: 0.5
 });

docs/solutions/models.md

@@ -41,7 +41,10 @@ one over the other.
 *   Face landmark model:
     [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
     [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   [Model card](https://mediapipe.page.link/facemesh-mc)
+*   Face landmark model w/ attention (aka Attention Mesh):
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite)
+*   [Model card](https://mediapipe.page.link/facemesh-mc),
+    [Model card (w/ attention)](https://mediapipe.page.link/attentionmesh-mc)
 
 ### [Iris](https://google.github.io/mediapipe/solutions/iris)
 

docs/solutions/objectron.md

@@ -338,11 +338,10 @@ with mp_objectron.Objectron(static_image_mode=False,
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Convert the BGR image to RGB.
-    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = objectron.process(image)
 
     # Draw the box landmarks on the image.
@@ -354,7 +353,8 @@ with mp_objectron.Objectron(static_image_mode=False,
               image, detected_object.landmarks_2d, mp_objectron.BOX_CONNECTIONS)
             mp_drawing.draw_axis(image, detected_object.rotation,
                                  detected_object.translation)
-    cv2.imshow('MediaPipe Objectron', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Objectron', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()

docs/solutions/pose.md

@@ -316,12 +316,10 @@ with mp_pose.Pose(
       # If loading a video, use 'break' instead of 'continue'.
       continue
 
-    # Flip the image horizontally for a later selfie-view display, and convert
-    # the BGR image to RGB.
-    image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
     # To improve performance, optionally mark the image as not writeable to
     # pass by reference.
     image.flags.writeable = False
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     results = pose.process(image)
 
     # Draw the pose annotation on the image.
@@ -332,7 +330,8 @@ with mp_pose.Pose(
         results.pose_landmarks,
         mp_pose.POSE_CONNECTIONS,
         landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
-    cv2.imshow('MediaPipe Pose', image)
+    # Flip the image horizontally for a selfie-view display.
+    cv2.imshow('MediaPipe Pose', cv2.flip(image, 1))
     if cv2.waitKey(5) & 0xFF == 27:
       break
 cap.release()