Project import generated by Copybara.

GitOrigin-RevId: f9a66589eaf652bb93f8e37ed9e4da26e59ef214

Dockerfile

@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=2.0.0
+ARG BAZEL_VERSION=3.4.1
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

README.md

@@ -8,15 +8,14 @@ nav_order: 1
 
 --------------------------------------------------------------------------------
 
-## Cross-platform ML solutions made simple
+## Live ML anywhere
 
-[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
-and developers to build world-class ML solutions and applications for mobile,
-desktop/cloud, web and IoT devices.
+[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
+ML solutions for live and streaming media.
 
 ![accelerated.png](docs/images/accelerated_small.png)                                                               | ![cross_platform.png](docs/images/cross_platform_small.png)
 :------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
-***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
+***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
 ![ready_to_use.png](docs/images/ready_to_use_small.png)                                                             | ![open_source.png](docs/images/open_source_small.png)
 ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework*            | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
 
@@ -50,6 +49,10 @@ Object Detection
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |         |     | ✅       |        |     |
 [YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m)                           |         |     | ✅       |        |     |
 
+See also
+[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
+for ML models released in MediaPipe.
+
 ## MediaPipe on the Web
 
 MediaPipe on the Web is an effort to run the same ML solutions built for mobile
@@ -89,7 +92,9 @@ run code search using
 
 ## Publications
 
-*   [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
+*   [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
+    in Google Developers Blog
+*   [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
     in Google Developers Blog
 *   [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
     in Google AI Blog

WORKSPACE

@@ -10,7 +10,7 @@ http_archive(
     sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "2.0.0")
+versions.check(minimum_bazel_version = "3.4.0")
 
 
 # ABSL cpp library lts_2020_02_25
@@ -324,8 +324,9 @@ maven_install(
         "androidx.lifecycle:lifecycle-common:2.2.0",
         "androidx.annotation:annotation:aar:1.1.0",
         "androidx.appcompat:appcompat:aar:1.1.0-rc01",
-        "androidx.camera:camera-core:aar:1.0.0-alpha06",
-        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
+        "androidx.camera:camera-core:1.0.0-beta10",
+        "androidx.camera:camera-camera2:1.0.0-beta10",
+        "androidx.camera:camera-lifecycle:1.0.0-beta10",
         "androidx.constraintlayout:constraintlayout:aar:1.1.3",
         "androidx.core:core:aar:1.1.0-rc03",
         "androidx.legacy:legacy-support-v4:aar:1.0.0",
@@ -337,6 +338,7 @@ maven_install(
         "com.google.flogger:flogger-system-backend:0.3.1",
         "com.google.flogger:flogger:0.3.1",
         "com.google.guava:guava:27.0.1-android",
+        "com.google.guava:listenablefuture:1.0",
         "junit:junit:4.12",
         "org.hamcrest:hamcrest-library:1.3",
     ],
@@ -362,13 +364,12 @@ http_archive(
 )
 
 #Tensorflow repo should always go after the other external dependencies.
-# 2020-05-11
-_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
-_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
+# 2020-08-30
+_TENSORFLOW_GIT_COMMIT = "57b009e31e59bd1a7ae85ef8c0232ed86c9b71db"
+_TENSORFLOW_SHA256= "de7f5f06204e057383028c7e53f3b352cdf85b3a40981b1a770c9a415a792c0e"
 http_archive(
     name = "org_tensorflow",
     urls = [
-      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
     patches = [

build_desktop_examples.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========================================================================
+#
+# Script to build/run all MediaPipe desktop example apps (with webcam input).
+#
+# To build and run all apps and store them in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir
+#   Omitting -d and the associated directory saves all generated apps in the
+#   current directory.
+# To build all apps and store them in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir -b
+#   Omitting -d and the associated directory saves all generated apps in the
+#   current directory.
+# To run all apps already stored in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir -r
+#   Omitting -d and the associated directory assumes all apps are in the current
+#   directory.
+
+set -e
+
+out_dir="."
+build_only=false
+run_only=false
+app_dir="mediapipe/examples/desktop"
+bin_dir="bazel-bin"
+declare -a default_bazel_flags=(build -c opt --define MEDIAPIPE_DISABLE_GPU=1)
+
+while [[ -n $1 ]]; do
+  case $1 in
+    -d)
+      shift
+      out_dir=$1
+      ;;
+    -b)
+      build_only=true
+      ;;
+    -r)
+      run_only=true
+      ;;
+    *)
+      echo "Unsupported input argument $1."
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+echo "app_dir: $app_dir"
+echo "out_dir: $out_dir"
+
+declare -a bazel_flags
+
+apps="${app_dir}/*"
+for app in ${apps}; do
+  if [[ -d "${app}" ]]; then
+    target_name=${app##*/}
+    if [[ "${target_name}" == "autoflip" ||
+          "${target_name}" == "hello_world" ||
+          "${target_name}" == "media_sequence" ||
+          "${target_name}" == "template_matching" ||
+          "${target_name}" == "youtube8m" ]]; then
+      continue
+    fi
+    target="${app}:${target_name}_cpu"
+
+    echo "=== Target: ${target}"
+
+    if [[ $run_only == false ]]; then
+      bazel_flags=("${default_bazel_flags[@]}")
+      bazel_flags+=(${target})
+
+      bazel "${bazel_flags[@]}"
+      cp -f "${bin_dir}/${app}/"*"_cpu" "${out_dir}"
+    fi
+    if [[ $build_only == false ]]; then
+      if  [[ ${target_name} == "multi_hand_tracking" ]]; then
+        graph_name="hand_tracking/multi_hand_tracking"
+      elif  [[ ${target_name} == "object_tracking" ]]; then
+        graph_name="tracking/object_detection_tracking"
+      elif [[ ${target_name} == "upper_body_pose_tracking" ]]; then
+        graph_name="pose_tracking/upper_body_pose_tracking"
+      else
+        graph_name="${target_name}/${target_name}"
+      fi
+      if [[ ${target_name} == "iris_tracking" ||
+            ${target_name} == "upper_body_pose_tracking" ]]; then
+        graph_suffix="cpu"
+      else
+        graph_suffix="desktop_live"
+      fi
+      GLOG_logtostderr=1 "${out_dir}/${target_name}_cpu" \
+        --calculator_graph_config_file=mediapipe/graphs/"${graph_name}_${graph_suffix}.pbtxt"
+    fi
+  fi
+done

docs/getting_started/hello_world_android.md

@@ -446,8 +446,8 @@ visible so that we can start seeing frames from the `previewFrameTexture`.
 However, before starting the camera, we need to decide which camera we want to
 use. [`CameraXPreviewHelper`] inherits from [`CameraHelper`] which provides two
 options, `FRONT` and `BACK`. We can pass in the decision from the `BUILD` file
-as metadata such that no code change is required to build a another version of
-the app using a different camera.
+as metadata such that no code change is required to build another version of the
+app using a different camera.
 
 Assuming we want to use `BACK` camera to perform edge detection on a live scene
 that we view from the camera, add the metadata into `AndroidManifest.xml`:

docs/getting_started/install.md

@@ -42,16 +42,16 @@ apps, see these [instructions](./building_examples.md#ios).
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install Bazel 2.0 or higher.
+    to install Bazel 3.4 or higher.
 
     For Nvidia Jetson and Raspberry Pi devices with ARM Ubuntu, Bazel needs to
     be built from source.
 
     ```bash
-    # For Bazel 3.0.0
-    wget https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-dist.zip
+    # For Bazel 3.4.0
+    wget https://github.com/bazelbuild/bazel/releases/download/3.4.0/bazel-3.4.0-dist.zip
     sudo apt-get install build-essential openjdk-8-jdk python zip unzip
-    unzip bazel-3.0.0-dist.zip
+    unzip bazel-3.4.0-dist.zip
     env EXTRA_BAZEL_ARGS="--host_javabase=@local_jdk//:jdk" bash ./compile.sh
     sudo cp output/bazel /usr/local/bin/
     ```
@@ -221,7 +221,7 @@ build issues.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel 2.0 or higher.
+    to install Bazel 3.4 or higher.
 
 3.  Install OpenCV.
 
@@ -356,7 +356,7 @@ build issues.
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install Bazel 2.0 or higher.
+    to install Bazel 3.4 or higher.
 
 4.  Install OpenCV and FFmpeg.
 
@@ -427,7 +427,6 @@ build issues.
         linkstatic = 1,
         visibility = ["//visibility:public"],
     )
-
     ```
 
 5.  Make sure that Python 3 and the Python "six" library are installed.
@@ -506,7 +505,7 @@ next section.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-windows.html)
-    to install Bazel 2.0 or higher.
+    to install Bazel 3.4 or higher.
 
 6.  Set Bazel variables.
 
@@ -567,7 +566,6 @@ next section.
     # I20200514 20:43:12.279618  1200 hello_world.cc:56] Hello World!
     # I20200514 20:43:12.279618  1200 hello_world.cc:56] Hello World!
     # I20200514 20:43:12.280613  1200 hello_world.cc:56] Hello World!
-
     ```
 
 If you run into a build error, please read
@@ -607,14 +605,14 @@ cameras. Alternatively, you use a video file as input.
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/3.0.0/release/bazel-3.0.0-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/3.0.0 && \
-    chmod 755 bazel-3.0.0-installer-linux-x86_64.sh && \
-    sudo ./bazel-3.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.0.0 && \
-    source /usr/local/bazel/3.0.0/lib/bazel/bin/bazel-complete.bash
+    https://storage.googleapis.com/bazel/3.4.0/release/bazel-3.4.0-installer-linux-x86_64.sh && \
+    sudo mkdir -p /usr/local/bazel/3.4.0 && \
+    chmod 755 bazel-3.4.0-installer-linux-x86_64.sh && \
+    sudo ./bazel-3.4.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/3.4.0 && \
+    source /usr/local/bazel/3.4.0/lib/bazel/bin/bazel-complete.bash
 
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.0.0/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/3.0.0/lib/bazel/bin/bazel'
+    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/3.4.0/lib/bazel/bin/bazel version && \
+    alias bazel='/usr/local/bazel/3.4.0/lib/bazel/bin/bazel'
     ```
 
 6.  Checkout MediaPipe repository.

docs/index.md

@@ -8,15 +8,14 @@ nav_order: 1
 
 --------------------------------------------------------------------------------
 
-## Cross-platform ML solutions made simple
+## Live ML anywhere
 
-[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
-and developers to build world-class ML solutions and applications for mobile,
-desktop/cloud, web and IoT devices.
+[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
+ML solutions for live and streaming media.
 
 ![accelerated.png](images/accelerated_small.png)                                                               | ![cross_platform.png](images/cross_platform_small.png)
 :------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
-***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
+***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
 ![ready_to_use.png](images/ready_to_use_small.png)                                                             | ![open_source.png](images/open_source_small.png)
 ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework*            | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
 
@@ -50,6 +49,10 @@ Object Detection
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |         |     | ✅       |        |     |
 [YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m)                           |         |     | ✅       |        |     |
 
+See also
+[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
+for ML models released in MediaPipe.
+
 ## MediaPipe on the Web
 
 MediaPipe on the Web is an effort to run the same ML solutions built for mobile
@@ -89,7 +92,9 @@ run code search using
 
 ## Publications
 
-*   [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
+*   [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
+    in Google Developers Blog
+*   [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
     in Google Developers Blog
 *   [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
     in Google AI Blog

docs/solutions/face_detection.md

@@ -102,9 +102,4 @@ to cross-compile and run MediaPipe examples on the
     [BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs](https://arxiv.org/abs/1907.05047)
     ([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
     ([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
-*   For front-facing/selfie camera:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
-    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
-*   For back-facing camera:
-    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
-*   [Model card](https://mediapipe.page.link/blazeface-mc)
+*   [Models and model cards](./models.md#face_detection)

docs/solutions/face_mesh.md

@@ -19,13 +19,18 @@ landmarks in real-time even on mobile devices. It employs machine learning (ML)
 to infer the 3D surface geometry, requiring only a single camera input without
 the need for a dedicated depth sensor. Utilizing lightweight model architectures
 together with GPU acceleration throughout the pipeline, the solution delivers
-real-time performance critical for live experiences. The core of the solution is
-the same as what powers
-[YouTube Stories](https://youtube-creators.googleblog.com/2018/11/introducing-more-ways-to-share-your.html)'
-creator effects, the
-[Augmented Faces API in ARCore](https://developers.google.com/ar/develop/java/augmented-faces/)
-and the
-[ML Kit Face Contour Detection API](https://firebase.google.com/docs/ml-kit/face-detection-concepts#contours).
+real-time performance critical for live experiences.
+
+Additionally, the solution is bundled with the Face Geometry module that bridges
+the gap between the face landmark estimation and useful real-time augmented
+reality (AR) applications. It establishes a metric 3D space and uses the face
+landmark screen positions to estimate face geometry within that space. The face
+geometry data consists of common 3D geometry primitives, including a face pose
+transformation matrix and a triangular face mesh. Under the hood, a lightweight
+statistical analysis method called
+[Procrustes Analysis](https://en.wikipedia.org/wiki/Procrustes_analysis) is
+employed to drive a robust, performant and portable logic. The analysis runs on
+CPU and has a minimal speed/memory footprint on top of the ML model inference.
 
 ![face_mesh_ar_effects.gif](../images/face_mesh_ar_effects.gif) |
 :-------------------------------------------------------------: |
@@ -67,15 +72,15 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-## Models
+### Models
 
-### Face Detection Model
+#### Face Detection Model
 
 The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
 model used in [MediaPipe Face Detection](./face_detection.md). Please refer to
 [MediaPipe Face Detection](./face_detection.md) for details.
 
-### Face Landmark Model
+#### Face Landmark Model
 
 For 3D face landmarks we employed transfer learning and trained a network with
 several objectives: the network simultaneously predicts 3D landmark coordinates
@@ -98,7 +103,108 @@ You can find more information about the face landmark model in this
 
 ![face_mesh_android_gpu.gif](../images/mobile/face_mesh_android_gpu.gif)   |
 :------------------------------------------------------------------------: |
-*Fig 2. Output of MediaPipe Face Mesh: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
+*Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
+
+## Face Geometry Module
+
+The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
+detection in the screen coordinate space: the X- and Y- coordinates are
+normalized screen coordinates, while the Z coordinate is relative and is scaled
+as the X coodinate under the
+[weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection).
+This format is well-suited for some applications, however it does not directly
+enable the full spectrum of augmented reality (AR) features like aligning a
+virtual 3D object with a detected face.
+
+The
+[Face Geometry module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
+moves away from the screen coordinate space towards a metric 3D space and
+provides necessary primitives to handle a detected face as a regular 3D object.
+By design, you'll be able to use a perspective camera to project the final 3D
+scene back into the screen coordinate space with a guarantee that the face
+landmark positions are not changed.
+
+### Key Concepts
+
+#### Metric 3D Space
+
+The **Metric 3D space** established within the Face Geometry module is a
+right-handed orthonormal metric 3D coordinate space. Within the space, there is
+a **virtual perspective camera** located at the space origin and pointed in the
+negative direction of the Z-axis. In the current pipeline, it is assumed that
+the input camera frames are observed by exactly this virtual camera and
+therefore its parameters are later used to convert the screen landmark
+coordinates back into the Metric 3D space. The *virtual camera parameters* can
+be set freely, however for better results it is advised to set them as close to
+the *real physical camera parameters* as possible.
+
+![face_geometry_metric_3d_space.gif](../images/face_geometry_metric_3d_space.gif) |
+:----------------------------------------------------------------------------: |
+*Fig 3. A visualization of multiple key elements in the Metric 3D space.*      |
+
+#### Canonical Face Model
+
+The **Canonical Face Model** is a static 3D model of a human face, which follows
+the 468 3D face landmark topology of the
+[Face Landmark Model](#face-landmark-model). The model bears two important
+functions:
+
+-   **Defines metric units**: the scale of the canonical face model defines the
+    metric units of the Metric 3D space. A metric unit used by the
+    [default canonical face model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx)
+    is a centimeter;
+-   **Bridges static and runtime spaces**: the face pose transformation matrix
+    is - in fact - a linear map from the canonical face model into the runtime
+    face landmark set estimated on each frame. This way, virtual 3D assets
+    modeled around the canonical face model can be aligned with a tracked face
+    by applying the face pose transformation matrix to them.
+
+### Components
+
+#### Geometry Pipeline
+
+The **Geometry Pipeline** is a key component, which is responsible for
+estimating face geometry objects within the Metric 3D space. On each frame, the
+following steps are executed in the given order:
+
+-   Face landmark screen coordinates are converted into the Metric 3D space
+    coordinates;
+-   Face pose transformation matrix is estimated as a rigid linear mapping from
+    the canonical face metric landmark set into the runtime face metric landmark
+    set in a way that minimizes a difference between the two;
+-   A face mesh is created using the runtime face metric landmarks as the vertex
+    positions (XYZ), while both the vertex texture coordinates (UV) and the
+    triangular topology are inherited from the canonical face model.
+
+The geometry pipeline is implemented as a MediaPipe
+[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
+For your convenience, the face geometry pipeline calculator is bundled together
+with the face landmark module into a unified MediaPipe
+[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
+The face geometry format is defined as a Protocol Buffer
+[message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
+
+#### Effect Renderer
+
+The **Effect Renderer** is a component, which serves as a working example of a
+face effect renderer. It targets the *OpenGL ES 2.0* API to enable a real-time
+performance on mobile devices and supports the following rendering modes:
+
+-   **3D object rendering mode**: a virtual object is aligned with a detected
+    face to emulate an object attached to the face (example: glasses);
+-   **Face mesh rendering mode**: a texture is stretched on top of the face mesh
+    surface to emulate a face painting technique.
+
+In both rendering modes, the face mesh is first rendered as an occluder straight
+into the depth buffer. This step helps to create a more believable effect via
+hiding invisible elements behind the face surface.
+
+The effect renderer is implemented as a MediaPipe
+[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc).
+
+| ![face_geometry_renderer.gif](../images/face_geometry_renderer.gif)     |
+| :---------------------------------------------------------------------: |
+| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
 
 ## Example Apps
 
@@ -111,7 +217,12 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Mobile
+### Face Landmark Example
+
+Face landmark example showcases real-time, cross-platform face landmark
+detection. For visual reference, please refer to *Fig. 2*.
+
+#### Mobile
 
 *   Graph:
     [`mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
@@ -127,7 +238,7 @@ it, for Android modify `NUM_FACES` in
 and for iOS modify `kNumFaces` in
 [FaceMeshGpuViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm).
 
-### Desktop
+#### Desktop
 
 *   Running on CPU
     *   Graph:
@@ -143,18 +254,35 @@ and for iOS modify `kNumFaces` in
 Tip: Maximum number of faces to detect/process is set to 1 by default. To change
 it, in the graph file modify the option of `ConstantSidePacketCalculator`.
 
+### Face Effect Example
+
+Face effect example showcases real-time mobile face effect application use case
+for the Face Mesh solution. To enable a better user experience, this example
+only works for a single face. For visual reference, please refer to *Fig. 4*.
+
+#### Mobile
+
+*   Graph:
+    [`mediapipe/graphs/face_effect/face_effect_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_effect/face_effect_gpu.pbtxt)
+*   Android target:
+    [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ccnaDnffEuIXriBZr2SK_Eu4FpO7K44s)
+    [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD)
+*   iOS target:
+    [`mediapipe/examples/ios/faceeffect`](http:/mediapipe/examples/ios/faceeffect/BUILD)
+
 ## Resources
 
 *   Google AI Blog:
     [Real-Time AR Self-Expression with Machine Learning](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
 *   TensorFlow Blog:
     [Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
+*   Google Developers Blog:
+    [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
 *   Paper:
     [Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
     ([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
-*   Face detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
-*   Face landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   [Model card](https://mediapipe.page.link/facemesh-mc)
+*   Canonical face model:
+    [FBX](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx),
+    [OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
+    [UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
+*   [Models and model cards](./models.md#face_mesh)

docs/solutions/hair_segmentation.md

@@ -54,5 +54,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
     [Real-time Hair segmentation and recoloring on Mobile GPUs](https://arxiv.org/abs/1907.06740)
     ([presentation](https://drive.google.com/file/d/1C8WYlWdDRNtU1_pYBvkkG5Z5wqYqf0yj/view))
     ([supplementary video](https://drive.google.com/file/d/1LPtM99Ch2ogyXYbDNpEqnUfhFq0TfLuf/view))
-*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
-*   [Model card](https://mediapipe.page.link/hairsegmentation-mc)
+*   [Models and model cards](./models.md#hair_segmentation)

docs/solutions/hands.md

@@ -226,10 +226,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
     ([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
-*   Palm detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
-*   Hand landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-*   [Model card](https://mediapipe.page.link/handmc)
+*   [Models and model cards](./models.md#hands)

docs/solutions/instant_motion_tracking.md

@@ -114,9 +114,9 @@ MediaPipe examples.
 
 ## Resources
 
-* Google Developers Blog:
-  [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
-* Google AI Blog:
-  [The Instant Motion Tracking Behind Motion Stills AR](https://ai.googleblog.com/2018/02/the-instant-motion-tracking-behind.html)
-* Paper:
-  [Instant Motion Tracking and Its Applications to Augmented Reality](https://arxiv.org/abs/1907.06796)
+*   Google Developers Blog:
+    [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
+*   Google AI Blog:
+    [The Instant Motion Tracking Behind Motion Stills AR](https://ai.googleblog.com/2018/02/the-instant-motion-tracking-behind.html)
+*   Paper:
+    [Instant Motion Tracking and Its Applications to Augmented Reality](https://arxiv.org/abs/1907.06796)

docs/solutions/iris.md

@@ -199,11 +199,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
     ([presentation](https://youtu.be/cIhXkiiapQI))
-*   Face detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
-*   Face landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   Iris landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
-*   [Model card](https://mediapipe.page.link/iris-mc)
+*   [Models and model cards](./models.md#iris)

docs/solutions/knift.md

@@ -139,7 +139,4 @@ to run regular TFLite inference.
 
 *   Google Developers Blog:
     [MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
-*   [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
-*   [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
-*   [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
-*   [Model card](https://mediapipe.page.link/knift-mc)
+*   [Models and model cards](./models.md#knift)

docs/solutions/models.md

@@ -0,0 +1,77 @@
+---
+layout: default
+title: Models and Model Cards
+parent: Solutions
+nav_order: 30
+---
+
+# MediaPipe Models and Model Cards
+{: .no_toc }
+
+1. TOC
+{:toc}
+---
+
+### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
+
+*   Face detection model for front-facing/selfie camera:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
+    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
+*   Face detection model for back-facing camera:
+    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
+*   [Model card](https://mediapipe.page.link/blazeface-mc)
+
+### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
+
+*   Face landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
+*   [Model card](https://mediapipe.page.link/facemesh-mc)
+
+### [Iris](https://google.github.io/mediapipe/solutions/iris)
+
+*   Iris landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
+*   [Model card](https://mediapipe.page.link/iris-mc)
+
+### [Hands](https://google.github.io/mediapipe/solutions/hands)
+
+*   Palm detection model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
+*   Hand landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
+*   [Model card](https://mediapipe.page.link/handmc)
+
+### [Pose](https://google.github.io/mediapipe/solutions/pose)
+
+*   Pose detection model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
+*   Upper-body pose landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
+*   [Model card](https://mediapipe.page.link/blazepose-mc)
+
+### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
+
+*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
+*   [Model card](https://mediapipe.page.link/hairsegmentation-mc)
+
+### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)
+
+*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
+*   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
+*   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
+*   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
+
+### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
+
+*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
+*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
+
+### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
+
+*   [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
+*   [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
+*   [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
+*   [Model card](https://mediapipe.page.link/knift-mc)

docs/solutions/object_detection.md

@@ -144,7 +144,4 @@ to cross-compile and run MediaPipe examples on the
 
 ## Resources
 
-*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
-*   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
-*   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
-*   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
+*   [Models and model cards](./models.md#object_detection)

docs/solutions/objectron.md

@@ -191,5 +191,4 @@ to visualize its associated subgraphs, please see
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
     ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
-*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
-*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
+*   [Models and model cards](./models.md#objectron)

docs/solutions/pose.md

@@ -88,10 +88,12 @@ hip midpoints.
 ### Pose Landmark Model (BlazePose Tracker)
 
 The landmark model currently included in MediaPipe Pose predicts the location of
-25 upper-body landmarks (see figure below), with three degrees of freedom each
-(x, y location and visibility), plus two virtual alignment keypoints. It shares
-the same architecture as the full-body version that predicts 33 landmarks,
-described in more detail in the
+25 upper-body landmarks (see figure below), each with `(x, y, z, visibility)`,
+plus two virtual alignment keypoints. Note that the `z` value should be
+discarded as the model is currently not fully trained to predict depth, but this
+is something we have on the roadmap. The model shares the same architecture as
+the full-body version that predicts 33 landmarks, described in more detail in
+the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 and in this [paper](https://arxiv.org/abs/2006.10204).
 
@@ -189,8 +191,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
     ([presentation](https://youtu.be/YPpUOTRn5tA))
-*   Pose detection model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
-*   Upper-body pose landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
-*   [Model card](https://mediapipe.page.link/blazepose-mc)
+*   [Models and model cards](./models.md#pose)

docs/solutions/solutions.md

@@ -32,3 +32,7 @@ has_toc: false
 [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip)                               |         |     | ✅       |        |     |
 [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence)                    |         |     | ✅       |        |     |
 [YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m)                           |         |     | ✅       |        |     |
+
+See also
+[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
+for ML models released in MediaPipe.

docs/tools/tracing_and_profiling.md

@@ -26,9 +26,10 @@ To enable tracing and profiling of a mediapipe graph:
    1. The profiling library must be linked to the framework.
    2. Tracing and profiling must be enabled in the graph configuration.
 
-The profiling library is linked to the framework by default.  If needed,
-the profiling library can be omitted from the framework using the bazel
-command line option: `--define MEDIAPIPE_PROFILING=0`.
+The profiling library is linked to the framework by default for Desktop.
+If needed, it can be omitted from the framework using the bazel command line
+option: `--define MEDIAPIPE_PROFILING=0`.  For other platforms, you can use the
+bazel command line option `--define MEDIAPIPE_PROFILING=1` to link it.
 
 To enable tracing and profiling, the `CalculatorGraphConfig` (in
 [calculator.proto](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator.proto))
@@ -38,6 +39,7 @@ is a simple setup that turns on tracing and keeps 100 seconds of timing events:
 ```
 profiler_config {
   trace_enabled: true
+  enable_profiler: true
   trace_log_interval_count: 200
 }
 ```
@@ -71,6 +73,9 @@ MediaPipe will emit data into a pre-specified directory:
 
     You can open the Download Container. Logs will be located in `application
     container/.xcappdata/AppData/Documents/`
+    If XCode shows empty content for the downloaded container file, you can
+    right click and select 'Show Package Contents' in Finder. Logs
+    will be located in 'AppData/Documents/'
 
     ![Windows Download Container](../images/visualizer/ios_download_container.png)
 
@@ -144,6 +149,7 @@ we record ten intervals of half a second each. This can be overridden by adding
     ```bash
     profiler_config {
       trace_enabled: true
+      enable_profiler: true
       trace_log_path: "/sdcard/profiles/"
     }
     ```

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -10,6 +10,7 @@
     "mediapipe/examples/ios/helloworld/BUILD",
     "mediapipe/examples/ios/facedetectioncpu/BUILD",
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
+    "mediapipe/examples/ios/faceeffect/BUILD",
     "mediapipe/examples/ios/facemeshgpu/BUILD",
     "mediapipe/examples/ios/handdetectiongpu/BUILD",
     "mediapipe/examples/ios/handtrackinggpu/BUILD",
@@ -23,6 +24,7 @@
     "//mediapipe/examples/ios/helloworld:HelloWorldApp",
     "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
+    "//mediapipe/examples/ios/faceeffect:FaceEffectApp",
     "//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
     "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
     "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
@@ -90,6 +92,8 @@
     "mediapipe/examples/ios/helloworld",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/faceeffect",
+    "mediapipe/examples/ios/faceeffect/Base.lproj",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",
     "mediapipe/examples/ios/iristrackinggpu",
@@ -110,6 +114,7 @@
     "mediapipe/graphs",
     "mediapipe/graphs/edge_detection",
     "mediapipe/graphs/face_detection",
+    "mediapipe/graphs/face_geometry",
     "mediapipe/graphs/hand_tracking",
     "mediapipe/graphs/object_detection",
     "mediapipe/graphs/pose_tracking",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -13,6 +13,7 @@
     "mediapipe/examples/ios",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/faceeffect",
     "mediapipe/examples/ios/facemeshgpu",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",

mediapipe/calculators/core/BUILD

@@ -13,181 +13,120 @@
 # limitations under the License.
 #
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 licenses(["notice"])
 
 package(default_visibility = ["//visibility:private"])
 
-proto_library(
+mediapipe_proto_library(
     name = "concatenate_vector_calculator_proto",
     srcs = ["concatenate_vector_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "dequantize_byte_array_calculator_proto",
     srcs = ["dequantize_byte_array_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_cloner_calculator_proto",
     srcs = ["packet_cloner_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_resampler_calculator_proto",
     srcs = ["packet_resampler_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_thinner_calculator_proto",
     srcs = ["packet_thinner_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "split_vector_calculator_proto",
     srcs = ["split_vector_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "quantize_float_vector_calculator_proto",
     srcs = ["quantize_float_vector_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "sequence_shift_calculator_proto",
     srcs = ["sequence_shift_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "gate_calculator_proto",
     srcs = ["gate_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "constant_side_packet_calculator_proto",
     srcs = ["constant_side_packet_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "clip_vector_size_calculator_proto",
     srcs = ["clip_vector_size_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_cloner_calculator_cc_proto",
-    srcs = ["packet_cloner_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":packet_cloner_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_resampler_calculator_cc_proto",
-    srcs = ["packet_resampler_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":packet_resampler_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_thinner_calculator_cc_proto",
-    srcs = ["packet_thinner_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":packet_thinner_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "split_vector_calculator_cc_proto",
-    srcs = ["split_vector_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":split_vector_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "concatenate_vector_calculator_cc_proto",
-    srcs = ["concatenate_vector_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":concatenate_vector_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "clip_vector_size_calculator_cc_proto",
-    srcs = ["clip_vector_size_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":clip_vector_size_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "dequantize_byte_array_calculator_cc_proto",
-    srcs = ["dequantize_byte_array_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":dequantize_byte_array_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "quantize_float_vector_calculator_cc_proto",
-    srcs = ["quantize_float_vector_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":quantize_float_vector_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "sequence_shift_calculator_cc_proto",
-    srcs = ["sequence_shift_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":sequence_shift_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "gate_calculator_cc_proto",
-    srcs = ["gate_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":gate_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "constant_side_packet_calculator_cc_proto",
-    srcs = ["constant_side_packet_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":constant_side_packet_calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
 cc_library(
@@ -286,6 +225,15 @@ cc_library(
     name = "concatenate_vector_calculator",
     srcs = ["concatenate_vector_calculator.cc"],
     hdrs = ["concatenate_vector_calculator.h"],
+    copts = select({
+        # Needed for "//mediapipe/framework/formats:tensor" compatibility on Apple
+        # platforms for Metal pulled in via the tensor.h header.
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -50,6 +50,9 @@ REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
 typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
 REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
 
+typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"

mediapipe/calculators/core/gate_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019-2020 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -134,7 +134,13 @@ class GateCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) final {
-    use_side_packet_for_allow_disallow_ = false;
+    const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
+    use_calculator_option_for_allow_disallow_ =
+        options.has_allowance_override();
+    if (use_calculator_option_for_allow_disallow_) {
+      allow_by_calculator_option_ = options.allowance_override();
+    }
+
     if (cc->InputSidePackets().HasTag("ALLOW")) {
       use_side_packet_for_allow_disallow_ = true;
       allow_by_side_packet_decision_ =
@@ -150,24 +156,27 @@ class GateCalculator : public CalculatorBase {
     last_gate_state_ = GATE_UNINITIALIZED;
     RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
 
-    const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
     empty_packets_as_allow_ = options.empty_packets_as_allow();
-
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) final {
-    bool allow = empty_packets_as_allow_;
-    if (use_side_packet_for_allow_disallow_) {
-      allow = allow_by_side_packet_decision_;
-    } else {
-      if (cc->Inputs().HasTag("ALLOW") &&
-          !cc->Inputs().Tag("ALLOW").IsEmpty()) {
-        allow = cc->Inputs().Tag("ALLOW").Get<bool>();
-      }
-      if (cc->Inputs().HasTag("DISALLOW") &&
-          !cc->Inputs().Tag("DISALLOW").IsEmpty()) {
-        allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
+    // The allow/disallow signal in the calculator option has the highest
+    // priority. If it's not set, use the stream/side packet signal.
+    bool allow = allow_by_calculator_option_;
+    if (!use_calculator_option_for_allow_disallow_) {
+      allow = empty_packets_as_allow_;
+      if (use_side_packet_for_allow_disallow_) {
+        allow = allow_by_side_packet_decision_;
+      } else {
+        if (cc->Inputs().HasTag("ALLOW") &&
+            !cc->Inputs().Tag("ALLOW").IsEmpty()) {
+          allow = cc->Inputs().Tag("ALLOW").Get<bool>();
+        }
+        if (cc->Inputs().HasTag("DISALLOW") &&
+            !cc->Inputs().Tag("DISALLOW").IsEmpty()) {
+          allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
+        }
       }
     }
     const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
@@ -203,9 +212,11 @@ class GateCalculator : public CalculatorBase {
  private:
   GateState last_gate_state_ = GATE_UNINITIALIZED;
   int num_data_streams_;
-  bool empty_packets_as_allow_;
-  bool use_side_packet_for_allow_disallow_;
-  bool allow_by_side_packet_decision_;
+  bool empty_packets_as_allow_ = false;
+  bool use_side_packet_for_allow_disallow_ = false;
+  bool allow_by_side_packet_decision_ = false;
+  bool use_calculator_option_for_allow_disallow_ = false;
+  bool allow_by_calculator_option_ = false;
 };
 REGISTER_CALCULATOR(GateCalculator);
 

mediapipe/calculators/core/gate_calculator.proto

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019-2020 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,4 +27,9 @@ message GateCalculatorOptions {
   // disallowing the corresponding packets in the data input streams. Setting
   // this option to true inverts that, allowing the data packets to go through.
   optional bool empty_packets_as_allow = 1;
+
+  // If set, the calculator will always allow (if set to yes) or disallow (if
+  // set to no) the input streams to pass through, and ignore the ALLOW or
+  // DISALLOW input stream or side input packets.
+  optional bool allowance_override = 2;
 }

mediapipe/calculators/core/gate_calculator_test.cc

@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2019-2020 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -330,5 +330,52 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
   ASSERT_EQ(0, output.size());
 }
 
+TEST_F(GateCalculatorTest,
+       TestCalculatorOptionDecisionOverrideOverStreamSingal) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "ALLOW:gating_stream"
+        output_stream: "test_output"
+        options: {
+          [mediapipe.GateCalculatorOptions.ext] {
+            allowance_override: false
+          }
+        }
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  // The CalculatorOptions says disallow and the stream says allow. Should
+  // follow the CalculatorOptions' decision to disallow outputting anything.
+  RunTimeStep(kTimestampValue0, "ALLOW", true);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(0, output.size());
+}
+
+TEST_F(GateCalculatorTest,
+       TestCalculatorOptionDecisionOverrideOverSidePacketSingal) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_side_packet: "ALLOW:gating_packet"
+        output_stream: "test_output"
+        options: {
+          [mediapipe.GateCalculatorOptions.ext] {
+            allowance_override: true
+          }
+        }
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  // The CalculatorOptions says allow and the side packet says disallow. Should
+  // follow the CalculatorOptions' decision to allow outputting a packet.
+  runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
+  RunTimeStep(kTimestampValue0, true);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(1, output.size());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/core/mux_calculator_test.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <memory>
+
 #include "mediapipe/calculators/core/split_vector_calculator.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
            kOutputName, output_fn);
   EXPECT_EQ(output, input_packets);
 }
+
+constexpr char kDualInputGraphConfig[] = R"proto(
+  input_stream: "input_0"
+  input_stream: "input_1"
+  input_stream: "input_select"
+  output_stream: "test_output"
+  node {
+    calculator: "MuxCalculator"
+    input_stream: "INPUT:0:input_0"
+    input_stream: "INPUT:1:input_1"
+    input_stream: "SELECT:input_select"
+    output_stream: "OUTPUT:test_output"
+  }
+)proto";
+
+TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          kDualInputGraphConfig);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  std::shared_ptr<int> output;
+  MP_ASSERT_OK(
+      graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
+        output = p.Get<std::shared_ptr<int>>();
+        return ::mediapipe::OkStatus();
+      }));
+
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto one = std::make_shared<int>(1);
+  auto two = std::make_shared<int>(2);
+  auto three = std::make_shared<int>(3);
+  std::weak_ptr<int> one_weak = one;
+  std::weak_ptr<int> two_weak = two;
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_0",
+      MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_1",
+      MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_1",
+      MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
+  EXPECT_EQ(one, nullptr);
+  EXPECT_EQ(two, nullptr);
+  EXPECT_EQ(three, nullptr);
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_select", MakePacket<int>(0).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(*output, 1);
+  EXPECT_NE(one_weak.lock(), nullptr);
+  EXPECT_EQ(two_weak.lock(), nullptr);
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_select", MakePacket<int>(1).At(Timestamp(1))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(*output, 3);
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -12,148 +12,78 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 licenses(["notice"])
 
 package(default_visibility = ["//visibility:private"])
 
-exports_files(["LICENSE"])
-
-proto_library(
+mediapipe_proto_library(
     name = "opencv_image_encoder_calculator_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
-    visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "scale_image_calculator_proto",
     srcs = ["scale_image_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework/formats:image_format_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "set_alpha_calculator_proto",
     srcs = ["set_alpha_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "image_cropping_calculator_proto",
     srcs = ["image_cropping_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "bilateral_filter_calculator_proto",
     srcs = ["bilateral_filter_calculator.proto"],
     visibility = [
         "//visibility:public",
     ],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "recolor_calculator_proto",
     srcs = ["recolor_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "opencv_image_encoder_calculator_cc_proto",
-    srcs = ["opencv_image_encoder_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = [
-        "//visibility:public",
-    ],
-    deps = [":opencv_image_encoder_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
-    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "mask_overlay_calculator_cc_proto",
-    srcs = ["mask_overlay_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":mask_overlay_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "scale_image_calculator_cc_proto",
-    srcs = ["scale_image_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/framework/formats:image_format_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":scale_image_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "set_alpha_calculator_cc_proto",
-    srcs = ["set_alpha_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":set_alpha_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "image_cropping_calculator_cc_proto",
-    srcs = ["image_cropping_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":image_cropping_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "bilateral_filter_calculator_cc_proto",
-    srcs = ["bilateral_filter_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = [
-        "//visibility:public",
-    ],
-    deps = [":bilateral_filter_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "recolor_calculator_cc_proto",
-    srcs = ["recolor_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":recolor_calculator_proto"],
-)
-
 cc_library(
     name = "color_convert_calculator",
     srcs = ["color_convert_calculator.cc"],
@@ -279,27 +209,17 @@ cc_library(
     alwayslink = 1,
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "image_transformation_calculator_proto",
     srcs = ["image_transformation_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/gpu:scale_mode_proto",
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "image_transformation_calculator_cc_proto",
-    srcs = ["image_transformation_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/gpu:scale_mode_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":image_transformation_calculator_proto"],
-)
-
 cc_library(
     name = "image_transformation_calculator",
     srcs = ["image_transformation_calculator.cc"],
@@ -560,32 +480,33 @@ cc_test(
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "mask_overlay_calculator_proto",
     srcs = ["mask_overlay_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "opencv_encoded_image_to_image_frame_calculator_proto",
     srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "feature_detector_calculator_proto",
     srcs = ["feature_detector_calculator.proto"],
-    deps = ["//mediapipe/framework:calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "feature_detector_calculator_cc_proto",
-    srcs = ["feature_detector_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":feature_detector_calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
 cc_library(

mediapipe/calculators/tensorflow/BUILD

@@ -311,7 +311,6 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
     }),
     alwayslink = 1,

mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc

@@ -42,16 +42,16 @@ namespace tf = tensorflow;
 // a flag controls whether a new first dimension is inserted before
 // concatenation.
 //
-// Currently, the number of tensors output will be buffer_size less than the
-// number of input tensors because no padding is implemented and only full
-// buffers are output.
+// The number of tensors output will be buffer_size less than the
+// number of input tensors unless padding is set to a non-zero value in the
+// options proto.
 //
 // The timestamp of the output batch will match the timestamp of the first
 // tensor in that batch by default. (e.g. when buffer_size frames are added, the
 // output tensor will have the timestamp of the first input.). This behavior can
 // be adjusted by the timestamp_offset option.
 //
-// Example config:
+// Example config without padding:
 // node {
 //   calculator: "LappedTensorBufferCalculator"
 //   input_stream: "input_tensor"
@@ -64,26 +64,50 @@ namespace tf = tensorflow;
 //     }
 //   }
 // }
+//
+// Example config with padding and timestamp output:
+// node {
+//   calculator: "LappedTensorBufferCalculator"
+//   input_stream: "input_tensor"
+//   output_stream: "output_tensor"
+//   output_stream: "output_timestamp"
+//   options {
+//     [mediapipe.LappedTensorBufferCalculatorOptions.ext] {
+//       buffer_size: 100
+//       overlap: 50
+//       add_batch_dim_to_tensors: true
+//       timestamp_offset: 25
+//       padding: 25
+//     }
+//   }
+// }
+
 class LappedTensorBufferCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
 
   ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
 
  private:
-  // Adds a batch dimension to the input tensor if specified in the calculator
-  // options.
+  // Adds a batch dimension to the input tensor if specified in the
+  // calculator options.
   ::mediapipe::Status AddBatchDimension(tf::Tensor* input_tensor);
+  // Sends the current buffer downstream.
+  ::mediapipe::Status ProcessBuffer(CalculatorContext* cc);
 
   int steps_until_output_;
   int buffer_size_;
   int overlap_;
   int timestamp_offset_;
+  int initialized_;
+
   std::unique_ptr<CircularBuffer<Timestamp>> timestamp_buffer_;
   std::unique_ptr<CircularBuffer<tf::Tensor>> buffer_;
   LappedTensorBufferCalculatorOptions options_;
 };
+
 REGISTER_CALCULATOR(LappedTensorBufferCalculator);
 
 ::mediapipe::Status LappedTensorBufferCalculator::GetContract(
@@ -93,8 +117,8 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
   cc->Inputs().Index(0).Set<tf::Tensor>(
       // tensorflow::Tensor stream.
   );
-  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
-      << "Only one output stream is supported.";
+  RET_CHECK_LE(cc->Outputs().NumEntries(), 2)
+      << "Only one or two output stream(s) is/are supported.";
 
   if (cc->InputSidePackets().HasTag(kBufferSize)) {
     cc->InputSidePackets().Tag(kBufferSize).Set<int>();
@@ -108,11 +132,15 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
   if (cc->InputSidePackets().HasTag(kCalculatorOptions)) {
     cc->InputSidePackets()
         .Tag(kCalculatorOptions)
-        .Set<LappedTensorBufferCalculatorOptions>();
+        .Set<LappedTensorBufferCalculator>();
   }
   cc->Outputs().Index(0).Set<tf::Tensor>(
       // Output tensorflow::Tensor stream with possibly overlapping steps.
   );
+  // Output timestamp stream with possibly overlapping steps.
+  if (cc->Outputs().NumEntries() > 1) {
+    cc->Outputs().Index(1).Set<std::vector<Timestamp>>();
+  }
   return ::mediapipe::OkStatus();
 }
 
@@ -141,10 +169,13 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
       << "Negative timestamp_offset is not allowed.";
   RET_CHECK_LT(timestamp_offset_, buffer_size_)
       << "output_frame_num_offset has to be less than buffer_size.";
+  RET_CHECK_LT(options_.padding(), buffer_size_)
+      << "padding option must be smaller than buffer size.";
   timestamp_buffer_ =
       absl::make_unique<CircularBuffer<Timestamp>>(buffer_size_);
   buffer_ = absl::make_unique<CircularBuffer<tf::Tensor>>(buffer_size_);
-  steps_until_output_ = buffer_size_;
+  steps_until_output_ = buffer_size_ - options_.padding();
+  initialized_ = false;
   return ::mediapipe::OkStatus();
 }
 
@@ -156,23 +187,36 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
   if (options_.add_batch_dim_to_tensors()) {
     RET_CHECK_OK(AddBatchDimension(&input_tensor));
   }
+  // Pad frames at the beginning with the first frame.
+  if (!initialized_) {
+    for (int i = 0; i < options_.padding(); ++i) {
+      buffer_->push_back(input_tensor);
+      timestamp_buffer_->push_back(cc->InputTimestamp());
+    }
+    initialized_ = true;
+  }
   buffer_->push_back(input_tensor);
   timestamp_buffer_->push_back(cc->InputTimestamp());
   --steps_until_output_;
-
   if (steps_until_output_ <= 0) {
-    auto concatenated = ::absl::make_unique<tf::Tensor>();
-
-    const tf::Status concat_status = tf::tensor::Concat(
-        std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
-        concatenated.get());
-    RET_CHECK(concat_status.ok()) << concat_status.ToString();
-
-    cc->Outputs().Index(0).Add(concatenated.release(),
-                               timestamp_buffer_->Get(timestamp_offset_));
-
-    steps_until_output_ = buffer_size_ - overlap_;
+    MP_RETURN_IF_ERROR(ProcessBuffer(cc));
   }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status LappedTensorBufferCalculator::Close(CalculatorContext* cc) {
+  if (!initialized_ || options_.padding() == 0) {
+    return ::mediapipe::OkStatus();
+  }
+  int last_frame = buffer_size_ - steps_until_output_ - 1;
+  const auto& pad_frame = buffer_->Get(last_frame);
+  for (int i = 0; i < steps_until_output_ + options_.padding(); ++i) {
+    buffer_->push_back(pad_frame);
+    timestamp_buffer_->push_back(cc->InputTimestamp());
+  }
+  MP_RETURN_IF_ERROR(ProcessBuffer(cc));
+
   return ::mediapipe::OkStatus();
 }
 
@@ -190,4 +234,29 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
   return ::mediapipe::OkStatus();
 }
 
+// Process buffer
+::mediapipe::Status LappedTensorBufferCalculator::ProcessBuffer(
+    CalculatorContext* cc) {
+  auto concatenated = ::absl::make_unique<tf::Tensor>();
+  const tf::Status concat_status = tf::tensor::Concat(
+      std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
+      concatenated.get());
+  RET_CHECK(concat_status.ok()) << concat_status.ToString();
+  // Output cancatenated tensor.
+  cc->Outputs().Index(0).Add(concatenated.release(),
+                             timestamp_buffer_->Get(timestamp_offset_));
+  if (cc->Outputs().NumEntries() > 1) {
+    auto output_timestamp = ::absl::make_unique<std::vector<Timestamp>>();
+    // Output timestamp vector.
+    *output_timestamp = std::vector<Timestamp>(timestamp_buffer_->begin(),
+                                               timestamp_buffer_->end());
+    RET_CHECK_EQ(output_timestamp->size(), buffer_size_)
+        << "Output timestamp size is not correct.";
+    cc->Outputs().Index(1).Add(output_timestamp.release(),
+                               timestamp_buffer_->Get(timestamp_offset_));
+  }
+  steps_until_output_ = buffer_size_ - overlap_;
+  return ::mediapipe::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.proto

@@ -45,4 +45,8 @@ message LappedTensorBufferCalculatorOptions {
   // This is useful for aligning the timestamp to be centered on the input
   // range.
   optional int32 timestamp_offset = 4 [default = 0];
+
+  // Amount of padding (repeating of first/last value) to add to the beginning
+  // and end of the input stream.
+  optional int32 padding = 5;
 }

mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator_test.cc

@@ -31,11 +31,15 @@ namespace tf = ::tensorflow;
 class LappedTensorBufferCalculatorTest : public ::testing::Test {
  protected:
   void SetUpCalculator(int buffer_size, int overlap, bool add_dim,
-                       int timestamp_offset) {
+                       int timestamp_offset, int padding,
+                       bool timestamp_output) {
     CalculatorGraphConfig::Node config;
     config.set_calculator("LappedTensorBufferCalculator");
     config.add_input_stream("input_tensor");
     config.add_output_stream("output_tensor");
+    if (timestamp_output) {
+      config.add_output_stream("output_timestamp");
+    }
     auto options = config.mutable_options()->MutableExtension(
         LappedTensorBufferCalculatorOptions::ext);
     options->set_buffer_size(buffer_size);
@@ -44,13 +48,14 @@ class LappedTensorBufferCalculatorTest : public ::testing::Test {
       options->set_add_batch_dim_to_tensors(true);
     }
     options->set_timestamp_offset(timestamp_offset);
+    options->set_padding(padding);
     runner_ = ::absl::make_unique<CalculatorRunner>(config);
   }
   std::unique_ptr<CalculatorRunner> runner_;
 };
 
 TEST_F(LappedTensorBufferCalculatorTest, OneToOne) {
-  SetUpCalculator(1, 0, false, 0);
+  SetUpCalculator(1, 0, false, 0, 0, false);
   int num_timesteps = 3;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -74,7 +79,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToTwo) {
   int buffer_size = 2;
   int overlap = 1;
   bool add_dim = false;
-  SetUpCalculator(buffer_size, overlap, add_dim, 0);
+  SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
   int num_timesteps = 3;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -100,7 +105,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThree) {
   int buffer_size = 3;
   int overlap = 2;
   bool add_dim = false;
-  SetUpCalculator(buffer_size, overlap, add_dim, 0);
+  SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
   int num_timesteps = 3;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -126,7 +131,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeSkip) {
   int buffer_size = 3;
   int overlap = 1;
   bool add_dim = false;
-  SetUpCalculator(buffer_size, overlap, add_dim, 0);
+  SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
   int num_timesteps = 3;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -152,7 +157,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatch) {
   int buffer_size = 3;
   int overlap = 2;
   bool add_dim = true;
-  SetUpCalculator(buffer_size, overlap, add_dim, 0);
+  SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
   int num_timesteps = 3;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -180,7 +185,7 @@ TEST_F(LappedTensorBufferCalculatorTest, NegativeTimestampOffsetFails) {
   int overlap = 15;
   bool add_dim = true;
   int timestamp_offset = -7;
-  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
+  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
   int num_timesteps = 20;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -197,7 +202,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OutOfRangeTimestampOffsetFails) {
   int overlap = 15;
   bool add_dim = true;
   int timestamp_offset = buffer_size;
-  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
+  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
   int num_timesteps = 20;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -214,7 +219,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
   int overlap = 15;
   bool add_dim = true;
   int timestamp_offset = 7;
-  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
+  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
   int num_timesteps = 20;
   for (int i = 0; i < num_timesteps; ++i) {
     auto input = ::absl::make_unique<tensorflow::Tensor>(
@@ -236,5 +241,37 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
   }
 }
 
+TEST_F(LappedTensorBufferCalculatorTest,
+       OneToThreeBatchTimestampOffsetPadding) {
+  int buffer_size = 12;
+  int overlap = 6;
+  bool add_dim = true;
+  int timestamp_offset = 3;
+  int padding = 0;
+  SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, padding,
+                  true);
+  int num_timesteps = 20;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto input = ::absl::make_unique<tensorflow::Tensor>(
+        tensorflow::DT_FLOAT, tensorflow::TensorShape({1}));
+    input->tensor<float, 1>()(0) = i;
+    runner_->MutableInputs()->Index(0).packets.push_back(
+        Adopt(input.release()).At(Timestamp(i)));
+  }
+  ASSERT_TRUE(runner_->Run().ok());
+
+  const int output_size = num_timesteps / buffer_size + 1;
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  ASSERT_EQ(output_size, output_packets.size());
+  for (int i = 0; i < output_size; ++i) {
+    int64 value = output_packets[i].Timestamp().Value();
+    ASSERT_EQ(i * overlap + timestamp_offset, value);
+  }
+  const std::vector<Packet>& output_timestamps =
+      runner_->Outputs().Index(1).packets;
+  ASSERT_EQ(output_size, output_timestamps.size());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -184,6 +184,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
       features_present_[tag] = false;
     }
 
+    replace_keypoints_ = false;
     if (cc->Options<PackMediaSequenceCalculatorOptions>()
             .replace_data_instead_of_append()) {
       for (const auto& tag : cc->Inputs().GetTags()) {
@@ -212,6 +213,15 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           }
           mpms::ClearBBox(key, sequence_.get());
           mpms::ClearBBoxTimestamp(key, sequence_.get());
+          mpms::ClearBBoxIsAnnotated(key, sequence_.get());
+          mpms::ClearBBoxNumRegions(key, sequence_.get());
+          mpms::ClearBBoxLabelString(key, sequence_.get());
+          mpms::ClearBBoxLabelIndex(key, sequence_.get());
+          mpms::ClearBBoxClassString(key, sequence_.get());
+          mpms::ClearBBoxClassIndex(key, sequence_.get());
+          mpms::ClearBBoxTrackString(key, sequence_.get());
+          mpms::ClearBBoxTrackIndex(key, sequence_.get());
+          mpms::ClearUnmodifiedBBoxTimestamp(key, sequence_.get());
         }
         if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
           std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) /
@@ -223,8 +233,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
         if (absl::StartsWith(tag, kKeypointsTag)) {
           std::string key =
               tag.substr(sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1);
-          mpms::ClearBBoxPoint(key, sequence_.get());
-          mpms::ClearBBoxTimestamp(key, sequence_.get());
+          replace_keypoints_ = true;
         }
       }
       if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) {
@@ -342,11 +351,25 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                 .Get<std::unordered_map<
                     std::string, std::vector<std::pair<float, float>>>>();
         for (const auto& pair : keypoints) {
-          mpms::AddBBoxTimestamp(mpms::merge_prefix(key, pair.first),
-                                 cc->InputTimestamp().Value(), sequence_.get());
-          mpms::AddBBoxPoint(mpms::merge_prefix(key, pair.first), pair.second,
-                             sequence_.get());
+          std::string prefix = mpms::merge_prefix(key, pair.first);
+          if (replace_keypoints_) {
+            mpms::ClearBBoxPoint(prefix, sequence_.get());
+            mpms::ClearBBoxTimestamp(prefix, sequence_.get());
+            mpms::ClearBBoxIsAnnotated(prefix, sequence_.get());
+            mpms::ClearBBoxNumRegions(prefix, sequence_.get());
+            mpms::ClearBBoxLabelString(prefix, sequence_.get());
+            mpms::ClearBBoxLabelIndex(prefix, sequence_.get());
+            mpms::ClearBBoxClassString(prefix, sequence_.get());
+            mpms::ClearBBoxClassIndex(prefix, sequence_.get());
+            mpms::ClearBBoxTrackString(prefix, sequence_.get());
+            mpms::ClearBBoxTrackIndex(prefix, sequence_.get());
+            mpms::ClearUnmodifiedBBoxTimestamp(prefix, sequence_.get());
+          }
+          mpms::AddBBoxTimestamp(prefix, cc->InputTimestamp().Value(),
+                                 sequence_.get());
+          mpms::AddBBoxPoint(prefix, pair.second, sequence_.get());
         }
+        replace_keypoints_ = false;
       }
       if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) &&
           !cc->Inputs().Tag(tag).IsEmpty()) {
@@ -475,6 +498,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
 
   std::unique_ptr<tf::SequenceExample> sequence_;
   std::map<std::string, bool> features_present_;
+  bool replace_keypoints_;
 };
 REGISTER_CALCULATOR(PackMediaSequenceCalculator);
 

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -839,5 +839,59 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
   ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 4), 50);
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
+  SetUpCalculator({"IMAGE:images", "BBOX:bbox"}, {}, false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
+  std::vector<uchar> bytes;
+  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  std::string test_image_string(bytes.begin(), bytes.end());
+  OpenCvImageEncoderCalculatorResults encoded_image;
+  encoded_image.set_encoded_image(test_image_string);
+  int height = 2;
+  int width = 2;
+  encoded_image.set_width(width);
+  encoded_image.set_height(height);
+
+  int num_images = 5;  // Timestamps: 10, 20, 30, 40, 50
+  for (int i = 0; i < num_images; ++i) {
+    auto image_ptr =
+        ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
+    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+        Adopt(image_ptr.release()).At(Timestamp(i)));
+  }
+
+  for (int i = 0; i < num_images; ++i) {
+    auto detections = ::absl::make_unique<::std::vector<Detection>>();
+    Detection detection;
+    detection = Detection();
+    detection.add_label("relative bbox");
+    detection.add_label_id(1);
+    detection.add_score(0.75);
+    Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
+        .ConvertToProto(detection.mutable_location_data());
+    detections->push_back(detection);
+    runner_->MutableInputs()->Tag("BBOX").packets.push_back(
+        Adopt(detections.release()).At(Timestamp(i)));
+  }
+
+  for (int i = 0; i < 10; ++i) {
+    mpms::AddBBoxTimestamp(-1, input_sequence.get());
+    mpms::AddBBoxIsAnnotated(-1, input_sequence.get());
+    mpms::AddBBoxNumRegions(-1, input_sequence.get());
+    mpms::AddBBoxLabelString({"anything"}, input_sequence.get());
+    mpms::AddBBoxLabelIndex({-1}, input_sequence.get());
+    mpms::AddBBoxClassString({"anything"}, input_sequence.get());
+    mpms::AddBBoxClassIndex({-1}, input_sequence.get());
+    mpms::AddBBoxTrackString({"anything"}, input_sequence.get());
+    mpms::AddBBoxTrackIndex({-1}, input_sequence.get());
+  }
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+  // If the all the previous values aren't cleared, this assert will fail.
+  MP_ASSERT_OK(runner_->Run());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -51,7 +51,7 @@ namespace {
 void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
   for (auto& node : *graph_def->mutable_node()) {
     if (node.device().empty()) {
-      node.set_device(device_id);
+      node.set_device(std::string(device_id));
     }
   }
 }

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc

@@ -47,7 +47,7 @@ namespace {
 void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) {
   for (auto& node : *graph_def->mutable_node()) {
     if (node.device().empty()) {
-      node.set_device(device_id);
+      node.set_device(std::string(device_id));
     }
   }
 }

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto

@@ -56,4 +56,11 @@ message UnpackMediaSequenceCalculatorOptions {
   // the clip start and end times and outputs these for the
   // AudioDecoderCalculator to consume.
   optional AudioDecoderOptions base_audio_decoder_options = 9;
+
+  optional string keypoint_names = 10 [
+    default =
+        "NOSE,LEFT_EAR,RIGHT_EAR,LEFT_SHOULDER,RIGHT_SHOULDER,LEFT_FORE_PAW,RIGHT_FORE_PAW,LEFT_HIP,RIGHT_HIP,LEFT_HIND_PAW,RIGHT_HIND_PAW,ROOT_TAIL"
+  ];
+  // When the keypoint doesn't exists, output this default value.
+  optional float default_keypoint_location = 11 [default = -1.0];
 }

mediapipe/calculators/tflite/BUILD

@@ -13,131 +13,91 @@
 # limitations under the License.
 #
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 load("@bazel_skylib//lib:selects.bzl", "selects")
 
 licenses(["notice"])
 
 package(default_visibility = ["//visibility:private"])
 
-proto_library(
+mediapipe_proto_library(
     name = "ssd_anchors_calculator_proto",
     srcs = ["ssd_anchors_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_custom_op_resolver_calculator_proto",
     srcs = ["tflite_custom_op_resolver_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_inference_calculator_proto",
     srcs = ["tflite_inference_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_converter_calculator_proto",
     srcs = ["tflite_converter_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_tensors_to_segmentation_calculator_proto",
     srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_tensors_to_detections_calculator_proto",
     srcs = ["tflite_tensors_to_detections_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_tensors_to_classification_calculator_proto",
     srcs = ["tflite_tensors_to_classification_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "tflite_tensors_to_landmarks_calculator_proto",
     srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "ssd_anchors_calculator_cc_proto",
-    srcs = ["ssd_anchors_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":ssd_anchors_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_custom_op_resolver_calculator_cc_proto",
-    srcs = ["tflite_custom_op_resolver_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_custom_op_resolver_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_converter_calculator_cc_proto",
-    srcs = ["tflite_converter_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_converter_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_tensors_to_segmentation_calculator_cc_proto",
-    srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_tensors_to_segmentation_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_inference_calculator_cc_proto",
-    srcs = ["tflite_inference_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_inference_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_tensors_to_detections_calculator_cc_proto",
-    srcs = ["tflite_tensors_to_detections_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_tensors_to_detections_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_tensors_to_classification_calculator_cc_proto",
-    srcs = ["tflite_tensors_to_classification_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_tensors_to_classification_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "tflite_tensors_to_landmarks_calculator_cc_proto",
-    srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":tflite_tensors_to_landmarks_calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
 cc_library(

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -281,6 +281,9 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   bool use_quantized_tensors_ = false;
 
   bool use_advanced_gpu_api_ = false;
+  bool allow_precision_loss_ = false;
+  ::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API
+      tflite_gpu_runner_api_;
 
   bool use_kernel_caching_ = false;
   std::string cached_kernel_filename_;
@@ -365,6 +368,8 @@ bool ShouldUseGpu(CC* cc) {
                           options.has_delegate() &&
                           options.delegate().has_gpu() &&
                           options.delegate().gpu().use_advanced_gpu_api();
+  allow_precision_loss_ = options.delegate().gpu().allow_precision_loss();
+  tflite_gpu_runner_api_ = options.delegate().gpu().api();
 
   use_kernel_caching_ =
       use_advanced_gpu_api_ && options.delegate().gpu().use_kernel_caching();
@@ -703,11 +708,23 @@ bool ShouldUseGpu(CC* cc) {
 
   // Create runner
   tflite::gpu::InferenceOptions options;
-  options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
+  options.priority1 = allow_precision_loss_
+                          ? tflite::gpu::InferencePriority::MIN_LATENCY
+                          : tflite::gpu::InferencePriority::MAX_PRECISION;
   options.priority2 = tflite::gpu::InferencePriority::AUTO;
   options.priority3 = tflite::gpu::InferencePriority::AUTO;
   options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
   tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
+  if (tflite_gpu_runner_api_ ==
+      ::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API::
+          TfLiteInferenceCalculatorOptions_Delegate_Gpu_API_OPENGL) {
+    tflite_gpu_runner_->ForceOpenGL();
+  }
+  if (tflite_gpu_runner_api_ ==
+      ::mediapipe::TfLiteInferenceCalculatorOptions_Delegate_Gpu_API::
+          TfLiteInferenceCalculatorOptions_Delegate_Gpu_API_OPENCL) {
+    tflite_gpu_runner_->ForceOpenCL();
+  }
   MP_RETURN_IF_ERROR(
       tflite_gpu_runner_->InitializeWithModel(model, op_resolver));
 

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -49,6 +49,20 @@ message TfLiteInferenceCalculatorOptions {
       //   delegate: { gpu { use_advanced_gpu_api: true } }
       optional bool use_advanced_gpu_api = 1 [default = false];
 
+      // This option is valid for TFLite GPU delegate API2 only,
+      // Choose any of available APIs to force running inference using it.
+      enum API {
+        ANY = 0;
+        OPENGL = 1;
+        OPENCL = 2;
+      }
+      optional API api = 4 [default = ANY];
+
+      // This option is valid for TFLite GPU delegate API2 only,
+      // Set to true to use 16-bit float precision. If max precision is needed,
+      // set to false for 32-bit float calculations only.
+      optional bool allow_precision_loss = 3 [default = true];
+
       // Load pre-compiled serialized binary cache to accelerate init process.
       // Only available for OpenCL delegate on Android.
       optional bool use_kernel_caching = 2 [default = false];

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -21,9 +21,9 @@
 namespace mediapipe {
 
 // A calculator for converting TFLite tensors from regression models into
-// landmarks. Note that if the landmarks in the tensor has more than 4
-// dimensions, only the first 4 dimensions will be converted to
-// [x,y,z, visibility].
+// landmarks. Note that if the landmarks in the tensor has more than 5
+// dimensions, only the first 5 dimensions will be converted to
+// [x,y,z, visibility, presence].
 //
 // Input:
 //  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
@@ -209,6 +209,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
     if (num_dimensions > 3) {
       landmark->set_visibility(raw_landmarks[offset + 3]);
     }
+    if (num_dimensions > 4) {
+      landmark->set_presence(raw_landmarks[offset + 4]);
+    }
   }
 
   // Output normalized landmarks if required.
@@ -223,6 +226,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
       norm_landmark->set_z(landmark.z() / options_.input_image_width() /
                            options_.normalize_z());
       norm_landmark->set_visibility(landmark.visibility());
+      norm_landmark->set_presence(landmark.presence());
     }
     cc->Outputs()
         .Tag("NORM_LANDMARKS")

mediapipe/calculators/util/BUILD

@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
-exports_files(["LICENSE"])
-
 cc_library(
     name = "alignment_points_to_rects_calculator",
     srcs = ["alignment_points_to_rects_calculator.cc"],
@@ -38,186 +36,95 @@ cc_library(
     alwayslink = 1,
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "annotation_overlay_calculator_proto",
     srcs = ["annotation_overlay_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "detection_label_id_to_text_calculator_proto",
     srcs = ["detection_label_id_to_text_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "timed_box_list_id_to_label_calculator_proto",
     srcs = ["timed_box_list_id_to_label_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "latency_proto",
     srcs = ["latency.proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "non_max_suppression_calculator_proto",
     srcs = ["non_max_suppression_calculator.proto"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:calculator_proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_frequency_proto",
     srcs = ["packet_frequency.proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_frequency_calculator_proto",
     srcs = ["packet_frequency_calculator.proto"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "packet_latency_calculator_proto",
     srcs = ["packet_latency_calculator.proto"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "collection_has_min_size_calculator_proto",
     srcs = ["collection_has_min_size_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "association_calculator_proto",
     srcs = ["association_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "annotation_overlay_calculator_cc_proto",
-    srcs = ["annotation_overlay_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":annotation_overlay_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "detection_label_id_to_text_calculator_cc_proto",
-    srcs = ["detection_label_id_to_text_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":detection_label_id_to_text_calculator_proto",
-    ],
-)
-
-mediapipe_cc_proto_library(
-    name = "timed_box_list_id_to_label_calculator_cc_proto",
-    srcs = ["timed_box_list_id_to_label_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [
-        ":timed_box_list_id_to_label_calculator_proto",
-    ],
-)
-
-mediapipe_cc_proto_library(
-    name = "latency_cc_proto",
-    srcs = ["latency.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [":latency_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "non_max_suppression_calculator_cc_proto",
-    srcs = ["non_max_suppression_calculator.proto"],
-    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":non_max_suppression_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_frequency_cc_proto",
-    srcs = ["packet_frequency.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [":packet_frequency_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_frequency_calculator_cc_proto",
-    srcs = ["packet_frequency_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [
-        ":packet_frequency_calculator_proto",
-    ],
-)
-
-mediapipe_cc_proto_library(
-    name = "packet_latency_calculator_cc_proto",
-    srcs = ["packet_latency_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [
-        ":packet_latency_calculator_proto",
-    ],
-)
-
-mediapipe_cc_proto_library(
-    name = "collection_has_min_size_calculator_cc_proto",
-    srcs = ["collection_has_min_size_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [":collection_has_min_size_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "association_calculator_cc_proto",
-    srcs = ["association_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [":association_calculator_proto"],
-)
-
 cc_library(
     name = "packet_frequency_calculator",
     srcs = ["packet_frequency_calculator.cc"],
@@ -341,9 +248,11 @@ cc_library(
         "@com_google_absl//absl/strings",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:video_stream_header",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
         "//mediapipe/util:annotation_renderer",
@@ -367,6 +276,7 @@ cc_library(
     deps = [
         ":detection_label_id_to_text_calculator_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
+        "@com_google_absl//absl/container:node_hash_map",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
@@ -434,16 +344,6 @@ cc_library(
     alwayslink = 1,
 )
 
-mediapipe_cc_proto_library(
-    name = "thresholding_calculator_cc_proto",
-    srcs = ["thresholding_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":thresholding_calculator_proto"],
-)
-
 cc_library(
     name = "thresholding_calculator",
     srcs = ["thresholding_calculator.cc"],
@@ -457,16 +357,6 @@ cc_library(
     alwayslink = 1,
 )
 
-mediapipe_cc_proto_library(
-    name = "landmarks_to_detection_calculator_cc_proto",
-    srcs = ["landmarks_to_detection_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":landmarks_to_detection_calculator_proto"],
-)
-
 cc_library(
     name = "landmarks_to_detection_calculator",
     srcs = ["landmarks_to_detection_calculator.cc"],
@@ -483,46 +373,6 @@ cc_library(
     alwayslink = 1,
 )
 
-mediapipe_cc_proto_library(
-    name = "detections_to_rects_calculator_cc_proto",
-    srcs = ["detections_to_rects_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":detections_to_rects_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "landmark_projection_calculator_cc_proto",
-    srcs = ["landmark_projection_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":landmark_projection_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "landmarks_to_floats_calculator_cc_proto",
-    srcs = ["landmarks_to_floats_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":landmarks_to_floats_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "rect_transformation_calculator_cc_proto",
-    srcs = ["rect_transformation_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":rect_transformation_calculator_proto"],
-)
-
 cc_library(
     name = "detections_to_rects_calculator",
     srcs = [
@@ -593,162 +443,140 @@ cc_test(
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "rect_to_render_data_calculator_proto",
     srcs = ["rect_to_render_data_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "rect_to_render_scale_calculator_proto",
     srcs = ["rect_to_render_scale_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "detections_to_render_data_calculator_proto",
     srcs = ["detections_to_render_data_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "landmarks_to_render_data_calculator_proto",
     srcs = ["landmarks_to_render_data_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "timed_box_list_to_render_data_calculator_proto",
     srcs = ["timed_box_list_to_render_data_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "labels_to_render_data_calculator_proto",
     srcs = ["labels_to_render_data_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "thresholding_calculator_proto",
     srcs = ["thresholding_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "detections_to_rects_calculator_proto",
     srcs = ["detections_to_rects_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "landmark_projection_calculator_proto",
     srcs = ["landmark_projection_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "landmarks_to_floats_calculator_proto",
     srcs = ["landmarks_to_floats_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "rect_transformation_calculator_proto",
     srcs = ["rect_transformation_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "landmarks_to_detection_calculator_proto",
     srcs = ["landmarks_to_detection_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/util:color_proto",
         "//mediapipe/util:render_data_proto",
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "rect_to_render_data_calculator_cc_proto",
-    srcs = ["rect_to_render_data_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-        "//mediapipe/util:render_data_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":rect_to_render_data_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "rect_to_render_scale_calculator_cc_proto",
-    srcs = ["rect_to_render_scale_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":rect_to_render_scale_calculator_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "detections_to_render_data_calculator_cc_proto",
-    srcs = ["detections_to_render_data_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-        "//mediapipe/util:render_data_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":detections_to_render_data_calculator_proto"],
-)
-
 cc_library(
     name = "detections_to_render_data_calculator",
     srcs = ["detections_to_render_data_calculator.cc"],
@@ -768,18 +596,6 @@ cc_library(
     alwayslink = 1,
 )
 
-mediapipe_cc_proto_library(
-    name = "landmarks_to_render_data_calculator_cc_proto",
-    srcs = ["landmarks_to_render_data_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-        "//mediapipe/util:render_data_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":landmarks_to_render_data_calculator_proto"],
-)
-
 cc_library(
     name = "landmarks_to_render_data_calculator",
     srcs = ["landmarks_to_render_data_calculator.cc"],
@@ -800,18 +616,6 @@ cc_library(
     alwayslink = 1,
 )
 
-mediapipe_cc_proto_library(
-    name = "timed_box_list_to_render_data_calculator_cc_proto",
-    srcs = ["timed_box_list_to_render_data_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-        "//mediapipe/util:render_data_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":timed_box_list_to_render_data_calculator_proto"],
-)
-
 cc_library(
     name = "timed_box_list_to_render_data_calculator",
     srcs = ["timed_box_list_to_render_data_calculator.cc"],
@@ -945,6 +749,32 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_smoothing_calculator_proto",
+    srcs = ["landmarks_smoothing_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_smoothing_calculator",
+    srcs = ["landmarks_smoothing_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util/filtering:relative_velocity_filter",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "landmarks_to_floats_calculator",
     srcs = ["landmarks_to_floats_calculator.cc"],
@@ -992,25 +822,16 @@ cc_test(
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "top_k_scores_calculator_proto",
     srcs = ["top_k_scores_calculator.proto"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "top_k_scores_calculator_cc_proto",
-    srcs = ["top_k_scores_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":top_k_scores_calculator_proto"],
-)
-
 cc_library(
     name = "top_k_scores_calculator",
     srcs = ["top_k_scores_calculator.cc"],
@@ -1056,17 +877,6 @@ cc_test(
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "labels_to_render_data_calculator_cc_proto",
-    srcs = ["labels_to_render_data_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":labels_to_render_data_calculator_proto"],
-)
-
 cc_library(
     name = "local_file_contents_calculator",
     srcs = ["local_file_contents_calculator.cc"],
@@ -1103,6 +913,7 @@ cc_library(
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@com_google_absl//absl/strings",

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -20,9 +20,11 @@
 #include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/video_stream_header.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 #include "mediapipe/util/annotation_renderer.h"
@@ -40,13 +42,9 @@ namespace mediapipe {
 
 namespace {
 
-constexpr char kInputFrameTag[] = "IMAGE";
-constexpr char kOutputFrameTag[] = "IMAGE";
-
-constexpr char kInputVectorTag[] = "VECTOR";
-
-constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
-constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
+constexpr char kVectorTag[] = "VECTOR";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kImageFrameTag[] = "IMAGE";
 
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 
@@ -57,12 +55,15 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
 // merges the annotation overlay with the image frame. As a result, drawing in
 // this color is not supported and it should be set to something unlikely used.
 constexpr uchar kAnnotationBackgroundColor = 2;  // Grayscale value.
+
+// Future Image type.
+inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; }
 }  // namespace
 
 // A calculator for rendering data on images.
 //
 // Inputs:
-//  1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
+//  1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer),
 //     containing the input image.
 //     If output is CPU, and input isn't provided, the renderer creates a
 //     blank canvas with the width, height and color provided in the options.
@@ -74,7 +75,8 @@ constexpr uchar kAnnotationBackgroundColor = 2;  // Grayscale value.
 //     input vector items. These input streams are tagged with "VECTOR".
 //
 // Output:
-//  1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
+//  1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer),
+//  Note: Output types should match their corresponding input stream type.
 //
 // For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
 // output format is the same as input except for GRAY8 where the output is in
@@ -133,14 +135,17 @@ class AnnotationOverlayCalculator : public CalculatorBase {
   ::mediapipe::Status CreateRenderTargetCpu(CalculatorContext* cc,
                                             std::unique_ptr<cv::Mat>& image_mat,
                                             ImageFormat::Format* target_format);
+  template <typename Type, const char* Tag>
   ::mediapipe::Status CreateRenderTargetGpu(
       CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat);
+  template <typename Type, const char* Tag>
   ::mediapipe::Status RenderToGpu(CalculatorContext* cc, uchar* overlay_image);
   ::mediapipe::Status RenderToCpu(CalculatorContext* cc,
                                   const ImageFormat::Format& target_format,
                                   uchar* data_image);
 
   ::mediapipe::Status GlRender(CalculatorContext* cc);
+  template <typename Type, const char* Tag>
   ::mediapipe::Status GlSetup(CalculatorContext* cc);
 
   // Options for the calculator.
@@ -172,24 +177,26 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag(kInputFrameTag) &&
-      cc->Inputs().HasTag(kInputFrameTagGpu)) {
+  if (cc->Inputs().HasTag(kImageFrameTag) &&
+      cc->Inputs().HasTag(kGpuBufferTag)) {
     return ::mediapipe::InternalError("Cannot have multiple input images.");
   }
-  if (cc->Inputs().HasTag(kInputFrameTagGpu) !=
-      cc->Outputs().HasTag(kOutputFrameTagGpu)) {
+  if (cc->Inputs().HasTag(kGpuBufferTag) !=
+      cc->Outputs().HasTag(kGpuBufferTag)) {
     return ::mediapipe::InternalError("GPU output must have GPU input.");
   }
 
-  // Input image to render onto copy of.
+  // Input image to render onto copy of. Should be same type as output.
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
-    cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
-    use_gpu |= true;
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
+    CHECK(cc->Outputs().HasTag(kGpuBufferTag));
+    use_gpu = true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag(kInputFrameTag)) {
-    cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+    CHECK(cc->Outputs().HasTag(kImageFrameTag));
   }
 
   // Data streams to render.
@@ -197,7 +204,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
        ++id) {
     auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
     std::string tag = tag_and_index.first;
-    if (tag == kInputVectorTag) {
+    if (tag == kVectorTag) {
       cc->Inputs().Get(id).Set<std::vector<RenderData>>();
     } else if (tag.empty()) {
       // Empty tag defaults to accepting a single object of RenderData type.
@@ -205,15 +212,14 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     }
   }
 
-  // Rendered image.
+  // Rendered image. Should be same type as input.
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Outputs().HasTag(kOutputFrameTagGpu)) {
-    cc->Outputs().Tag(kOutputFrameTagGpu).Set<mediapipe::GpuBuffer>();
-    use_gpu |= true;
+  if (cc->Outputs().HasTag(kGpuBufferTag)) {
+    cc->Outputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Outputs().HasTag(kOutputFrameTag)) {
-    cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
   if (use_gpu) {
@@ -229,20 +235,16 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   cc->SetOffset(TimestampDiff(0));
 
   options_ = cc->Options<AnnotationOverlayCalculatorOptions>();
-  if (cc->Inputs().HasTag(kInputFrameTagGpu) &&
-      cc->Outputs().HasTag(kOutputFrameTagGpu)) {
+  if (cc->Inputs().HasTag(kGpuBufferTag) || HasImageTag(cc)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     use_gpu_ = true;
-#else
-    RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  if (cc->Inputs().HasTag(kInputFrameTagGpu) ||
-      cc->Inputs().HasTag(kInputFrameTag)) {
+  if (cc->Inputs().HasTag(kGpuBufferTag) ||
+      cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) {
     image_frame_available_ = true;
   } else {
-    image_frame_available_ = false;
     RET_CHECK(options_.has_canvas_width_px());
     RET_CHECK(options_.has_canvas_height_px());
   }
@@ -253,14 +255,12 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor());
 
   // Set the output header based on the input header (if present).
-  const char* input_tag = use_gpu_ ? kInputFrameTagGpu : kInputFrameTag;
-  const char* output_tag = use_gpu_ ? kOutputFrameTagGpu : kOutputFrameTag;
-  if (image_frame_available_ &&
-      !cc->Inputs().Tag(input_tag).Header().IsEmpty()) {
+  const char* tag = use_gpu_ ? kGpuBufferTag : kImageFrameTag;
+  if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) {
     const auto& input_header =
-        cc->Inputs().Tag(input_tag).Header().Get<VideoHeader>();
+        cc->Inputs().Tag(tag).Header().Get<VideoHeader>();
     auto* output_video_header = new VideoHeader(input_header);
-    cc->Outputs().Tag(output_tag).SetHeader(Adopt(output_video_header));
+    cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header));
   }
 
   if (use_gpu_) {
@@ -282,15 +282,20 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     if (!gpu_initialized_) {
       MP_RETURN_IF_ERROR(
           gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
-            MP_RETURN_IF_ERROR(GlSetup(cc));
-            return ::mediapipe::OkStatus();
+            return GlSetup<mediapipe::GpuBuffer, kGpuBufferTag>(cc);
           }));
       gpu_initialized_ = true;
     }
+    if (cc->Inputs().HasTag(kGpuBufferTag)) {
+      MP_RETURN_IF_ERROR(
+          (CreateRenderTargetGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
+              cc, image_mat)));
+    }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-    MP_RETURN_IF_ERROR(CreateRenderTargetGpu(cc, image_mat));
   } else {
-    MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+    if (cc->Inputs().HasTag(kImageFrameTag)) {
+      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+    }
   }
 
   // Reset the renderer with the image_mat. No copy here.
@@ -301,7 +306,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
        ++id) {
     auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
     std::string tag = tag_and_index.first;
-    if (!tag.empty() && tag != kInputVectorTag) {
+    if (!tag.empty() && tag != kVectorTag) {
       continue;
     }
     if (cc->Inputs().Get(id).IsEmpty()) {
@@ -312,7 +317,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
       const RenderData& render_data = cc->Inputs().Get(id).Get<RenderData>();
       renderer_->RenderDataOnImage(render_data);
     } else {
-      RET_CHECK_EQ(kInputVectorTag, tag);
+      RET_CHECK_EQ(kVectorTag, tag);
       const std::vector<RenderData>& render_data_vec =
           cc->Inputs().Get(id).Get<std::vector<RenderData>>();
       for (const RenderData& render_data : render_data_vec) {
@@ -327,8 +332,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     uchar* image_mat_ptr = image_mat->data;
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, cc, image_mat_ptr]() -> ::mediapipe::Status {
-          MP_RETURN_IF_ERROR(RenderToGpu(cc, image_mat_ptr));
-          return ::mediapipe::OkStatus();
+          return RenderToGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
+              cc, image_mat_ptr);
         }));
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
@@ -369,19 +374,21 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
                               ImageFrame::kDefaultAlignmentBoundary);
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  cc->Outputs()
-      .Tag(kOutputFrameTag)
-      .Add(output_frame.release(), cc->InputTimestamp());
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
+    cc->Outputs()
+        .Tag(kImageFrameTag)
+        .Add(output_frame.release(), cc->InputTimestamp());
+  }
 
   return ::mediapipe::OkStatus();
 }
 
+template <typename Type, const char* Tag>
 ::mediapipe::Status AnnotationOverlayCalculator::RenderToGpu(
     CalculatorContext* cc, uchar* overlay_image) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   // Source and destination textures.
-  const auto& input_frame =
-      cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
+  const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
   auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
 
   auto output_texture = gpu_helper_.CreateDestinationTexture(
@@ -414,10 +421,8 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   }
 
   // Send out blended image as GPU packet.
-  auto output_frame = output_texture.GetFrame<mediapipe::GpuBuffer>();
-  cc->Outputs()
-      .Tag(kOutputFrameTagGpu)
-      .Add(output_frame.release(), cc->InputTimestamp());
+  auto output_frame = output_texture.GetFrame<Type>();
+  cc->Outputs().Tag(Tag).Add(output_frame.release(), cc->InputTimestamp());
 
   // Cleanup
   input_texture.Release();
@@ -432,7 +437,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     ImageFormat::Format* target_format) {
   if (image_frame_available_) {
     const auto& input_frame =
-        cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
+        cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
 
     int target_mat_type;
     switch (input_frame.Format()) {
@@ -455,39 +460,32 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
 
     image_mat = absl::make_unique<cv::Mat>(
         input_frame.Height(), input_frame.Width(), target_mat_type);
+
+    auto input_mat = formats::MatView(&input_frame);
     if (input_frame.Format() == ImageFormat::GRAY8) {
-      const int target_num_channels =
-          ImageFrame::NumberOfChannelsForFormat(*target_format);
-      for (int i = 0; i < input_frame.PixelDataSize(); i++) {
-        const auto& pix = input_frame.PixelData()[i];
-        for (int c = 0; c < target_num_channels; c++) {
-          image_mat->data[i * target_num_channels + c] = pix;
-        }
-      }
+      cv::Mat rgb_mat;
+      cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
+      rgb_mat.copyTo(*image_mat);
     } else {
-      // Make of a copy since the input frame may be consumed by other nodes.
-      const int buffer_size =
-          input_frame.Height() * input_frame.Width() *
-          ImageFrame::NumberOfChannelsForFormat(*target_format);
-      input_frame.CopyToBuffer(image_mat->data, buffer_size);
+      input_mat.copyTo(*image_mat);
     }
   } else {
     image_mat = absl::make_unique<cv::Mat>(
         options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
         cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(),
                    options_.canvas_color().b()));
+    *target_format = ImageFormat::SRGB;
   }
 
   return ::mediapipe::OkStatus();
 }
 
+template <typename Type, const char* Tag>
 ::mediapipe::Status AnnotationOverlayCalculator::CreateRenderTargetGpu(
     CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   if (image_frame_available_) {
-    const auto& input_frame =
-        cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
-
+    const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
     const mediapipe::ImageFormat::Format format =
         mediapipe::ImageFormatForGpuBufferFormat(input_frame.format());
     if (format != mediapipe::ImageFormat::SRGBA &&
@@ -563,6 +561,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   return ::mediapipe::OkStatus();
 }
 
+template <typename Type, const char* Tag>
 ::mediapipe::Status AnnotationOverlayCalculator::GlSetup(
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -638,8 +637,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   const float alignment = ImageFrame::kGlDefaultAlignmentBoundary;
   const float scale_factor = options_.gpu_scale_factor();
   if (image_frame_available_) {
-    const auto& input_frame =
-        cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
+    const auto& input_frame = cc->Inputs().Tag(Tag).Get<Type>();
     width_ = RoundUp(input_frame.width(), alignment);
     height_ = RoundUp(input_frame.height(), alignment);
   } else {

mediapipe/calculators/util/detection_label_id_to_text_calculator.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/container/node_hash_map.h"
 #include "mediapipe/calculators/util/detection_label_id_to_text_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/detection.pb.h"
@@ -52,7 +53,7 @@ class DetectionLabelIdToTextCalculator : public CalculatorBase {
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
-  std::unordered_map<int, std::string> label_map_;
+  absl::node_hash_map<int, std::string> label_map_;
 };
 REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator);
 

mediapipe/calculators/util/filter_collection_calculator.cc

@@ -20,9 +20,14 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
 
 namespace mediapipe {
 
+typedef FilterCollectionCalculator<std::vector<uint64>>
+    FilterUInt64CollectionCalculator;
+REGISTER_CALCULATOR(FilterUInt64CollectionCalculator);
+
 typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
     FilterNormalizedRectCollectionCalculator;
 REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);

mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc

@@ -131,6 +131,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
         new_landmark->set_z(new_z);
         // Keep visibility as is.
         new_landmark->set_visibility(landmark.visibility());
+        // Keep presence as is.
+        new_landmark->set_presence(landmark.presence());
       }
 
       cc->Outputs().Get(output_id).AddPacket(

mediapipe/calculators/util/landmark_projection_calculator.cc

@@ -131,6 +131,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
         new_landmark->set_z(new_z);
         // Keep visibility as is.
         new_landmark->set_visibility(landmark.visibility());
+        // Keep presence as is.
+        new_landmark->set_presence(landmark.presence());
       }
 
       cc->Outputs().Get(output_id).AddPacket(

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -13,12 +13,12 @@
 // limitations under the License.
 
 #include "absl/algorithm/container.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
-#include "mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.pb.h"
-#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
 
 namespace mediapipe {
 
@@ -38,17 +38,17 @@ using ::mediapipe::RelativeVelocityFilter;
 // with sides parallel to axis.
 float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
                      int image_height) {
-  const auto& [lm_min_x, lm_max_x] = absl::c_minmax_element(
+  const auto& lm_minmax_x = absl::c_minmax_element(
       landmarks.landmark(),
       [](const auto& a, const auto& b) { return a.x() < b.x(); });
-  const float x_min = lm_min_x->x();
-  const float x_max = lm_max_x->x();
+  const float x_min = lm_minmax_x.first->x();
+  const float x_max = lm_minmax_x.second->x();
 
-  const auto& [lm_min_y, lm_max_y] = absl::c_minmax_element(
+  const auto& lm_minmax_y = absl::c_minmax_element(
       landmarks.landmark(),
       [](const auto& a, const auto& b) { return a.y() < b.y(); });
-  const float y_min = lm_min_y->y();
-  const float y_max = lm_max_y->y();
+  const float y_min = lm_minmax_y.first->y();
+  const float y_max = lm_minmax_y.second->y();
 
   const float object_width = (x_max - x_min) * image_width;
   const float object_height = (y_max - y_min) * image_height;
@@ -137,6 +137,8 @@ class VelocityFilter : public LandmarksFilter {
                           image_width);
       // Keep visibility as is.
       out_landmark->set_visibility(in_landmark.visibility());
+      // Keep presence as is.
+      out_landmark->set_presence(in_landmark.presence());
     }
 
     return ::mediapipe::OkStatus();
@@ -191,8 +193,8 @@ class VelocityFilter : public LandmarksFilter {
 //     input_stream: "NORM_LANDMARKS:pose_landmarks"
 //     input_stream: "IMAGE_SIZE:image_size"
 //     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
-//     node_options: {
-//       [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
 //         velocity_filter: {
 //           window_size: 5
 //           velocity_scale: 10.0

mediapipe/calculators/util/landmarks_smoothing_calculator.proto

@@ -16,7 +16,7 @@ syntax = "proto2";
 
 package mediapipe;
 
-import "mediapipe/framework/calculator.proto";
+import "mediapipe/framework/calculator_options.proto";
 
 message LandmarksSmoothingCalculatorOptions {
   extend CalculatorOptions {

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -33,7 +33,6 @@ constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
 constexpr char kRenderScaleTag[] = "RENDER_SCALE";
 constexpr char kRenderDataTag[] = "RENDER_DATA";
 constexpr char kLandmarkLabel[] = "KEYPOINT";
-constexpr int kMaxLandmarkThickness = 18;
 
 inline void SetColor(RenderAnnotation* annotation, const Color& color) {
   annotation->mutable_color()->set_r(color.r());
@@ -59,15 +58,18 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
 }
 
 void SetColorSizeValueFromZ(float z, float z_min, float z_max,
-                            RenderAnnotation* render_annotation) {
+                            RenderAnnotation* render_annotation,
+                            float min_depth_circle_thickness,
+                            float max_depth_circle_thickness) {
   const int color_value = 255 - static_cast<int>(Remap(z, z_min, z_max, 255));
   ::mediapipe::Color color;
   color.set_r(color_value);
   color.set_g(color_value);
   color.set_b(color_value);
   SetColor(render_annotation, color);
-  const int thickness = static_cast<int>((1.f - Remap(z, z_min, z_max, 1)) *
-                                         kMaxLandmarkThickness);
+  const float scale = max_depth_circle_thickness - min_depth_circle_thickness;
+  const int thickness = static_cast<int>(
+      min_depth_circle_thickness + (1.f - Remap(z, z_min, z_max, 1)) * scale);
   render_annotation->set_thickness(thickness);
 }
 
@@ -97,14 +99,19 @@ template <class LandmarkListType, class LandmarkType>
 void AddConnectionsWithDepth(const LandmarkListType& landmarks,
                              const std::vector<int>& landmark_connections,
                              bool utilize_visibility,
-                             float visibility_threshold, float thickness,
+                             float visibility_threshold, bool utilize_presence,
+                             float presence_threshold, float thickness,
                              bool normalized, float min_z, float max_z,
                              RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
-                                 ld1.visibility() < visibility_threshold)) {
+    if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
+                               ld1.visibility() < visibility_threshold)) {
+      continue;
+    }
+    if (utilize_presence && (ld0.presence() < presence_threshold ||
+                             ld1.presence() < presence_threshold)) {
       continue;
     }
     const int gray_val1 =
@@ -136,13 +143,18 @@ template <class LandmarkListType, class LandmarkType>
 void AddConnections(const LandmarkListType& landmarks,
                     const std::vector<int>& landmark_connections,
                     bool utilize_visibility, float visibility_threshold,
+                    bool utilize_presence, float presence_threshold,
                     const Color& connection_color, float thickness,
                     bool normalized, RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
-                                 ld1.visibility() < visibility_threshold)) {
+    if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
+                               ld1.visibility() < visibility_threshold)) {
+      continue;
+    }
+    if (utilize_presence && (ld0.presence() < presence_threshold ||
+                             ld1.presence() < presence_threshold)) {
       continue;
     }
     AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
@@ -238,13 +250,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
     if (visualize_depth) {
       AddConnectionsWithDepth<LandmarkList, Landmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
-          options_.visibility_threshold(), thickness, /*normalized=*/false,
-          z_min, z_max, render_data.get());
+          options_.visibility_threshold(), options_.utilize_presence(),
+          options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
+          z_max, render_data.get());
     } else {
       AddConnections<LandmarkList, Landmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
-          options_.visibility_threshold(), options_.connection_color(),
-          thickness, /*normalized=*/false, render_data.get());
+          options_.visibility_threshold(), options_.utilize_presence(),
+          options_.presence_threshold(), options_.connection_color(), thickness,
+          /*normalized=*/false, render_data.get());
     }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
@@ -254,11 +268,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
         continue;
       }
 
+      if (options_.utilize_presence() &&
+          landmark.presence() < options_.presence_threshold()) {
+        continue;
+      }
+
       auto* landmark_data_render = AddPointRenderData(
           options_.landmark_color(), thickness, render_data.get());
       if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
-                               landmark_data_render);
+        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
+                               options_.min_depth_circle_thickness(),
+                               options_.max_depth_circle_thickness());
       }
       auto* landmark_data = landmark_data_render->mutable_point();
       landmark_data->set_normalized(false);
@@ -279,13 +299,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
     if (visualize_depth) {
       AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
-          options_.visibility_threshold(), thickness, /*normalized=*/true,
-          z_min, z_max, render_data.get());
+          options_.visibility_threshold(), options_.utilize_presence(),
+          options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
+          z_max, render_data.get());
     } else {
       AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
           landmarks, landmark_connections_, options_.utilize_visibility(),
-          options_.visibility_threshold(), options_.connection_color(),
-          thickness, /*normalized=*/true, render_data.get());
+          options_.visibility_threshold(), options_.utilize_presence(),
+          options_.presence_threshold(), options_.connection_color(), thickness,
+          /*normalized=*/true, render_data.get());
     }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
@@ -294,12 +316,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
           landmark.visibility() < options_.visibility_threshold()) {
         continue;
       }
+      if (options_.utilize_presence() &&
+          landmark.presence() < options_.presence_threshold()) {
+        continue;
+      }
 
       auto* landmark_data_render = AddPointRenderData(
           options_.landmark_color(), thickness, render_data.get());
       if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
-                               landmark_data_render);
+        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
+                               options_.min_depth_circle_thickness(),
+                               options_.max_depth_circle_thickness());
       }
       auto* landmark_data = landmark_data_render->mutable_point();
       landmark_data->set_normalized(true);

mediapipe/calculators/util/landmarks_to_render_data_calculator.proto

@@ -49,4 +49,19 @@ message LandmarksToRenderDataCalculatorOptions {
   // Threshold to determine visibility of the landmark. Landmark with visibility
   // greater or equal than threshold is considered visible.
   optional double visibility_threshold = 7 [default = 0.0];
+
+  // Use landmarks presence while rendering landmarks and connections. If
+  // landmark is not present, neither it nor adjacent connections will be
+  // rendered.
+  optional bool utilize_presence = 8 [default = false];
+
+  // Threshold to determine presence of the landmark. Landmark with presence
+  // greater or equal than threshold is considered present.
+  optional double presence_threshold = 9 [default = 0.0];
+
+  // Min thickness of the drawing for landmark circle.
+  optional double min_depth_circle_thickness = 10 [default = 0.0];
+
+  // Max thickness of the drawing for landmark circle.
+  optional double max_depth_circle_thickness = 11 [default = 18.0];
 }

mediapipe/calculators/video/BUILD

@@ -317,6 +317,7 @@ cc_library(
         "//mediapipe/util/tracking:box_tracker",
         "//mediapipe/util/tracking:tracking_visualization_utilities",
         "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/container:node_hash_map",
         "@com_google_absl//absl/container:node_hash_set",
         "@com_google_absl//absl/strings",
     ],

mediapipe/calculators/video/box_tracker_calculator.cc

@@ -19,6 +19,7 @@
 #include <unordered_set>
 
 #include "absl/container/flat_hash_set.h"
+#include "absl/container/node_hash_map.h"
 #include "absl/container/node_hash_set.h"
 #include "absl/strings/numbers.h"
 #include "mediapipe/calculators/video/box_tracker_calculator.pb.h"
@@ -207,7 +208,7 @@ class BoxTrackerCalculator : public CalculatorBase {
   // Boxes that are tracked in streaming mode.
   MotionBoxMap streaming_motion_boxes_;
 
-  std::unordered_map<int, std::pair<TimedBox, TimedBox>> last_tracked_boxes_;
+  absl::node_hash_map<int, std::pair<TimedBox, TimedBox>> last_tracked_boxes_;
   int frame_num_since_reset_ = 0;
 
   // Cache used during streaming mode for fast forward tracking.

mediapipe/calculators/video/tool/BUILD

@@ -19,8 +19,6 @@ licenses(["notice"])
 
 package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"])
 
-exports_files(["LICENSE"])
-
 proto_library(
     name = "flow_quantizer_model_proto",
     srcs = ["flow_quantizer_model.proto"],

mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java

@@ -134,6 +134,9 @@ public class MainActivity extends AppCompatActivity {
   protected void onPause() {
     super.onPause();
     converter.close();
+
+    // Hide preview display until we re-open the camera again.
+    previewDisplayView.setVisibility(View.GONE);
   }
 
   @Override

mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD

@@ -0,0 +1,67 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+android_binary(
+    name = "faceeffect",
+    srcs = glob(["*.java"]),
+    assets = [
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    assets_dir = "",
+    manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.apps.faceeffect",
+        "appName": "Face Effect",
+        "mainActivity": ".MainActivity",
+        "cameraFacingFront": "True",
+        "binaryGraphName": "face_effect_gpu.binarypb",
+        "inputVideoStreamName": "input_video",
+        "outputVideoStreamName": "output_video",
+        "flipFramesVertically": "True",
+    },
+    multidex = "native",
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
+        "//mediapipe/framework/formats:matrix_data_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java

@@ -0,0 +1,176 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.faceeffect;
+
+import android.graphics.Color;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.GestureDetector;
+import android.view.Gravity;
+import android.view.MotionEvent;
+import android.view.View;
+import android.view.ViewGroup;
+import android.view.ViewGroup.LayoutParams;
+import android.widget.RelativeLayout;
+import android.widget.TextView;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry;
+import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData;
+import java.util.List;
+
+/** Main activity of MediaPipe face mesh app. */
+public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
+  private static final String TAG = "MainActivity";
+
+  // Stream names.
+  private static final String IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME =
+      "is_facepaint_effect_selected";
+  private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry";
+
+  private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!";
+
+  private static final int MATRIX_TRANSLATION_Z_INDEX = 14;
+
+  private final Object isFacepaintEffectSelectedLock = new Object();
+  private boolean isFacepaintEffectSelected;
+
+  private View effectSwitchingHintView;
+  private GestureDetector tapGestureDetector;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    // Add an effect switching hint view to the preview layout.
+    effectSwitchingHintView = createEffectSwitchingHintView();
+    effectSwitchingHintView.setVisibility(View.INVISIBLE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(effectSwitchingHintView);
+
+    // By default, render the glasses effect.
+    isFacepaintEffectSelected = false;
+
+    // This callback demonstrates how the output face geometry packet can be obtained and used
+    // in an Android app. As an example, the Z-translation component of the face pose transform
+    // matrix is logged for each face being equal to the approximate distance away from the camera
+    // in centimeters.
+    processor.addPacketCallback(
+        OUTPUT_FACE_GEOMETRY_STREAM_NAME,
+        (packet) -> {
+          effectSwitchingHintView.post(
+              new Runnable() {
+                @Override
+                public void run() {
+                  effectSwitchingHintView.setVisibility(View.VISIBLE);
+                }
+              });
+
+          Log.d(TAG, "Received a multi face geometry packet.");
+          List<FaceGeometry> multiFaceGeometry =
+              PacketGetter.getProtoVector(packet, FaceGeometry.parser());
+
+          StringBuilder approxDistanceAwayFromCameraLogMessage = new StringBuilder();
+          for (FaceGeometry faceGeometry : multiFaceGeometry) {
+            if (approxDistanceAwayFromCameraLogMessage.length() > 0) {
+              approxDistanceAwayFromCameraLogMessage.append(' ');
+            }
+            MatrixData poseTransformMatrix = faceGeometry.getPoseTransformMatrix();
+            approxDistanceAwayFromCameraLogMessage.append(
+                -poseTransformMatrix.getPackedData(MATRIX_TRANSLATION_Z_INDEX));
+          }
+
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] size = "
+                  + multiFaceGeometry.size()
+                  + "; approx. distance away from camera in cm for faces = ["
+                  + approxDistanceAwayFromCameraLogMessage
+                  + "]");
+        });
+
+    // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+    // packet to indicate which effect should be rendered on this frame.
+    processor.setOnWillAddFrameListener(
+        (timestamp) -> {
+          Packet isFacepaintEffectSelectedPacket = null;
+          try {
+            synchronized (isFacepaintEffectSelectedLock) {
+              isFacepaintEffectSelectedPacket =
+                  processor.getPacketCreator().createBool(isFacepaintEffectSelected);
+            }
+
+            processor
+                .getGraph()
+                .addPacketToInputStream(
+                    IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME,
+                    isFacepaintEffectSelectedPacket,
+                    timestamp);
+          } catch (RuntimeException e) {
+            Log.e(
+                TAG,
+                "Exception while adding packet to input stream while switching effects: " + e);
+          } finally {
+            if (isFacepaintEffectSelectedPacket != null) {
+              isFacepaintEffectSelectedPacket.release();
+            }
+          }
+        });
+
+    // We use the tap gesture detector to switch between face effects. This allows users to try
+    // multiple pre-bundled face effects without a need to recompile the app.
+    tapGestureDetector =
+        new GestureDetector(
+            this,
+            new GestureDetector.SimpleOnGestureListener() {
+              @Override
+              public void onLongPress(MotionEvent event) {
+                switchEffect();
+              }
+
+              @Override
+              public boolean onSingleTapUp(MotionEvent event) {
+                switchEffect();
+                return true;
+              }
+
+              private void switchEffect() {
+                synchronized (isFacepaintEffectSelectedLock) {
+                  isFacepaintEffectSelected = !isFacepaintEffectSelected;
+                }
+              }
+            });
+  }
+
+  @Override
+  public boolean onTouchEvent(MotionEvent event) {
+    return tapGestureDetector.onTouchEvent(event);
+  }
+
+  private View createEffectSwitchingHintView() {
+    TextView effectSwitchingHintView = new TextView(getApplicationContext());
+    effectSwitchingHintView.setLayoutParams(
+        new RelativeLayout.LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.FILL_PARENT));
+    effectSwitchingHintView.setText(EFFECT_SWITCHING_HINT_TEXT);
+    effectSwitchingHintView.setGravity(Gravity.CENTER_HORIZONTAL | Gravity.BOTTOM);
+    effectSwitchingHintView.setPadding(0, 0, 0, 480);
+    effectSwitchingHintView.setTextColor(Color.parseColor("#ffffff"));
+    effectSwitchingHintView.setTextSize((float) 24);
+
+    return effectSwitchingHintView;
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/MainActivity.java

@@ -32,16 +32,22 @@ public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
   private static final String FOCAL_LENGTH_STREAM_NAME = "focal_length_pixel";
   private static final String OUTPUT_LANDMARKS_STREAM_NAME = "face_landmarks_with_iris";
 
+  private boolean haveAddedSidePackets = false;
+
   @Override
   protected void onCameraStarted(SurfaceTexture surfaceTexture) {
     super.onCameraStarted(surfaceTexture);
 
-    float focalLength = cameraHelper.getFocalLengthPixels();
-    if (focalLength != Float.MIN_VALUE) {
-      Packet focalLengthSidePacket = processor.getPacketCreator().createFloat32(focalLength);
-      Map<String, Packet> inputSidePackets = new HashMap<>();
-      inputSidePackets.put(FOCAL_LENGTH_STREAM_NAME, focalLengthSidePacket);
-      processor.setInputSidePackets(inputSidePackets);
+    // onCameraStarted gets called each time the activity resumes, but we only want to do this once.
+    if (!haveAddedSidePackets) {
+      float focalLength = cameraHelper.getFocalLengthPixels();
+      if (focalLength != Float.MIN_VALUE) {
+        Packet focalLengthSidePacket = processor.getPacketCreator().createFloat32(focalLength);
+        Map<String, Packet> inputSidePackets = new HashMap<>();
+        inputSidePackets.put(FOCAL_LENGTH_STREAM_NAME, focalLengthSidePacket);
+        processor.setInputSidePackets(inputSidePackets);
+      }
+      haveAddedSidePackets = true;
     }
   }
 

mediapipe/examples/coral/Dockerfile

@@ -62,7 +62,7 @@ COPY . /mediapipe/
 
 # Install bazel
 # Please match the current MediaPipe Bazel requirements according to docs.
-ARG BAZEL_VERSION=2.0.0
+ARG BAZEL_VERSION=3.4.1
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
     wget --no-check-certificate -O  /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc

@@ -35,7 +35,7 @@ constexpr char kDetectedBorders[] = "BORDERS";
 constexpr char kCropRect[] = "CROP_RECT";
 // Field-of-view (degrees) of the camera's x-axis (width).
 // TODO: Parameterize FOV based on camera specs.
-constexpr float kWidthFieldOfView = 60;
+constexpr float kFieldOfView = 60;
 
 namespace mediapipe {
 namespace autoflip {
@@ -244,6 +244,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
     frame_width_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
     frame_height_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
   } else if (cc->Inputs().HasTag(kVideoSize)) {
+    if (cc->Inputs().Tag(kVideoSize).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     frame_width_ =
         cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
     frame_height_ =
@@ -256,13 +259,13 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
   if (!initialized_) {
     path_solver_height_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_zoom(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
     path_solver_width_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_pan(), 0, frame_width_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_width_) / kFieldOfView);
     path_solver_offset_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_tilt(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
     max_frame_value_ = 1.0;
     target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
     // If target size is set and wider than input aspect, make sure to always
@@ -302,6 +305,14 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
   }
 
   if (cc->Inputs().HasTag(kDetections)) {
+    if (cc->Inputs().Tag(kDetections).IsEmpty()) {
+      auto default_rect = absl::make_unique<mediapipe::Rect>();
+      default_rect->set_width(frame_width_);
+      default_rect->set_height(frame_height_);
+      cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
+                                       Timestamp(cc->InputTimestamp()));
+      return ::mediapipe::OkStatus();
+    }
     auto raw_detections =
         cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
     for (const auto& detection : raw_detections) {
@@ -339,15 +350,24 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
     offset_y = last_measured_y_offset_;
   }
 
-  // Compute smoothed camera paths.
+  // Compute smoothed zoom camera path.
   MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
       height, cc->InputTimestamp().Microseconds()));
+  int path_height;
+  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
+  int path_width = path_height * target_aspect_;
+
+  // Update pixel-per-degree value for pan/tilt.
+  MP_RETURN_IF_ERROR(path_solver_width_->UpdatePixelsPerDegree(
+      static_cast<float>(path_width) / kFieldOfView));
+  MP_RETURN_IF_ERROR(path_solver_offset_->UpdatePixelsPerDegree(
+      static_cast<float>(path_height) / kFieldOfView));
+
+  // Compute smoothed pan/tilt paths.
   MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
       offset_x, cc->InputTimestamp().Microseconds()));
   MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
       offset_y, cc->InputTimestamp().Microseconds()));
-  int path_height;
-  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
   int path_offset_x;
   MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
   int path_offset_y;
@@ -359,7 +379,7 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
   } else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
     path_offset_y = frame_height_ - ceil(path_height / 2.0);
   }
-  int path_width = path_height * target_aspect_;
+
   if (path_offset_x - ceil(path_width / 2.0) < 0) {
     path_offset_x = ceil(path_width / 2.0);
   } else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc

@@ -174,15 +174,15 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
       ContentZoomingCalculatorOptions::ext);
   options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_pan()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);
   AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
   AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
   MP_ASSERT_OK(runner->Run());
   CheckCropRect(450, 550, 111, 111, 0,
                 runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(488, 550, 111, 111, 1,
+  CheckCropRect(483, 550, 111, 111, 1,
                 runner->Outputs().Tag("CROP_RECT").packets);
 }
 
@@ -190,17 +190,17 @@ TEST(ContentZoomingCalculatorTest, TiltConfig) {
   auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
   auto* options = config.mutable_options()->MutableExtension(
       ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
   options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);
   AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
   AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
   MP_ASSERT_OK(runner->Run());
   CheckCropRect(450, 550, 111, 111, 0,
                 runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(450, 588, 111, 111, 1,
+  CheckCropRect(450, 583, 111, 111, 1,
                 runner->Outputs().Tag("CROP_RECT").packets);
 }
 
@@ -208,8 +208,8 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
   auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
   auto* options = config.mutable_options()->MutableExtension(
       ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
   options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);
@@ -418,6 +418,25 @@ TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
                 runner->Outputs().Tag("CROP_RECT").packets);
 }
 
+TEST(ContentZoomingCalculatorTest, EmptySize) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  MP_ASSERT_OK(runner->Run());
+  ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0);
+}
+
+TEST(ContentZoomingCalculatorTest, EmptyDetections) {
+  auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
+  auto runner = ::absl::make_unique<CalculatorRunner>(config);
+  auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
+  runner->MutableInputs()
+      ->Tag("VIDEO_SIZE")
+      .packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
+  MP_ASSERT_OK(runner->Run());
+  CheckCropRect(0, 0, 1000, 1000, 0,
+                runner->Outputs().Tag("CROP_RECT").packets);
+}
+
 }  // namespace
 }  // namespace autoflip
 

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc

@@ -87,5 +87,13 @@ namespace autoflip {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status KinematicPathSolver::UpdatePixelsPerDegree(
+    const float pixels_per_degree) {
+  RET_CHECK_GT(pixels_per_degree_, 0)
+      << "pixels_per_degree must be larger than 0.";
+  pixels_per_degree_ = pixels_per_degree;
+  return ::mediapipe::OkStatus();
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h

@@ -46,6 +46,8 @@ class KinematicPathSolver {
   ::mediapipe::Status UpdatePrediction(const int64 time_us);
   // Get the state at a time.
   ::mediapipe::Status GetState(int* position);
+  // Update PixelPerDegree value.
+  ::mediapipe::Status UpdatePixelsPerDegree(const float pixels_per_degree);
 
  private:
   // Tuning options.

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc

@@ -207,6 +207,28 @@ TEST(KinematicPathSolverTest, PassMaxVelocity) {
   EXPECT_EQ(state, 600);
 }
 
+TEST(KinematicPathSolverTest, PassDegPerPxChange) {
+  KinematicOptions options;
+  // Set min motion to 2deg
+  options.set_min_motion_to_reframe(2.0);
+  options.set_update_rate(1);
+  options.set_max_velocity(1000);
+  // Set degrees / pixel to 16.6
+  KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
+  int state;
+  MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
+  // Move target by 20px / 16.6 = 1.2deg
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to not move.
+  EXPECT_EQ(state, 500);
+  MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to move.
+  EXPECT_EQ(state, 516);
+}
+
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt

@@ -1,5 +1,5 @@
 # MediaPipe graph that performs face detection with TensorFlow Lite on CPU.  Model paths setup for web use.
-# TODO: parameterize input paths to support desktop use.
+# TODO: parameterize input paths to support desktop use, for web only.
 input_stream: "VIDEO:input_video"
 output_stream: "DETECTIONS:output_detections"
 
@@ -37,7 +37,7 @@ node {
   output_stream: "TENSORS:detection_tensors"
   options: {
     [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/models/face_detection_front.tflite"
+      model_path: "face_detection_front.tflite"
     }
   }
 }
@@ -118,7 +118,7 @@ node {
   output_stream: "labeled_detections"
   options: {
     [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
-      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+      label_map_path: "face_detection_front_labelmap.txt"
     }
   }
 }

mediapipe/examples/ios/faceeffect/AppDelegate.h

@@ -0,0 +1,21 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

mediapipe/examples/ios/faceeffect/AppDelegate.m

@@ -0,0 +1,59 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end

mediapipe/examples/ios/faceeffect/BUILD

@@ -0,0 +1,92 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+load(
+    "//mediapipe/examples/ios:bundle_id.bzl",
+    "BUNDLE_ID_PREFIX",
+    "example_provisioning",
+)
+
+licenses(["notice"])
+
+MIN_IOS_VERSION = "10.0"
+
+alias(
+    name = "faceeffect",
+    actual = "FaceEffectApp",
+)
+
+ios_application(
+    name = "FaceEffectApp",
+    app_icons = ["//mediapipe/examples/ios/common:AppIcon"],
+    bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = example_provisioning(),
+    deps = [
+        ":FaceEffectAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "FaceEffectAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "FaceEffectViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "FaceEffectViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:matrix_data_cc_proto",
+            "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+            "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        ],
+    }),
+)

mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="16096" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina5_5" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="16086"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="FaceEffectViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="736"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="414" height="716"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="76" y="283" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Tap to switch between effects!" textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="b2C-zd-Zba" userLabel="Effect Switching Label">
+                                        <rect key="frame" x="77" y="488" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_effectSwitchingHintLabel" destination="b2C-zd-Zba" id="Uvx-2n-l74"/>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/faceeffect/FaceEffectViewController.h

@@ -0,0 +1,19 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface FaceEffectViewController : UIViewController
+
+@end

mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm

@@ -0,0 +1,254 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "FaceEffectViewController.h"
+
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+#include <utility>
+
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+
+static NSString* const kGraphName = @"face_effect_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kIsFacepaintEffectSelectedInputStream = "is_facepaint_effect_selected";
+static const char* kOutputStream = "output_video";
+static const char* kMultiFaceGeometryStream = "multi_face_geometry";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+static const int kMatrixTranslationZIndex = 14;
+
+@interface FaceEffectViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* graph;
+
+@end
+
+@implementation FaceEffectViewController {
+  /// Handle tap gestures.
+  UITapGestureRecognizer* _tapGestureRecognizer;
+  BOOL _isFacepaintEffectSelected;
+
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Inform the user about how to switch between effects.
+  UILabel* _effectSwitchingHintLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.graph.delegate = nil;
+  [self.graph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.graph closeAllInputStreamsWithError:nil];
+  [self.graph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _effectSwitchingHintLabel.hidden = YES;
+  _tapGestureRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
+                                                                  action:@selector(handleTap)];
+  [self.view addGestureRecognizer:_tapGestureRecognizer];
+
+  // By default, render the glasses effect.
+  _isFacepaintEffectSelected = NO;
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer insertSublayer:_renderer.layer atIndex:0];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  _renderer.mirrored = NO;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+  _cameraSource.videoMirrored = YES;
+
+  self.graph = [[self class] loadGraphFromResource:kGraphName];
+  self.graph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.graph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.graph.
+  NSError* error;
+  if (![self.graph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - UITapGestureRecognizer methods
+
+// We use the tap gesture recognizer to switch between face effects. This allows users to try
+// multiple pre-bundled face effects without a need to recompile the app.
+- (void)handleTap {
+  dispatch_async(_videoQueue, ^{
+    _isFacepaintEffectSelected = !_isFacepaintEffectSelected;
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      _effectSwitchingHintLabel.hidden = NO;
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+//
+// This callback demonstrates how the output face geometry packet can be obtained and used in an
+// iOS app. As an example, the Z-translation component of the face pose transform matrix is logged
+// for each face being equal to the approximate distance away from the camera in centimeters.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kMultiFaceGeometryStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face geometry", packet.Timestamp().Value());
+      return;
+    }
+
+    const auto& multiFaceGeometry =
+        packet.Get<std::vector<::mediapipe::face_geometry::FaceGeometry>>();
+    NSLog(@"[TS:%lld] Number of face instances with geometry: %lu ", packet.Timestamp().Value(),
+          multiFaceGeometry.size());
+    for (int faceIndex = 0; faceIndex < multiFaceGeometry.size(); ++faceIndex) {
+      const auto& faceGeometry = multiFaceGeometry[faceIndex];
+      NSLog(@"\tApprox. distance away from camera for face[%d]: %.6f cm", faceIndex,
+            -faceGeometry.pose_transform_matrix().packed_data(kMatrixTranslationZIndex));
+    }
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+
+  mediapipe::Timestamp graphTimestamp(static_cast<mediapipe::TimestampBaseType>(
+      mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp)));
+
+  mediapipe::Packet isFacepaintEffectSelectedPacket =
+      mediapipe::MakePacket<bool>(_isFacepaintEffectSelected).At(graphTimestamp);
+
+  [self.graph sendPixelBuffer:imageBuffer
+                   intoStream:kInputStream
+                   packetType:MPPPacketTypePixelBuffer
+                    timestamp:graphTimestamp];
+
+  // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+  // packet to indicate which effect should be rendered on this frame.
+  [self.graph movePacket:std::move(isFacepaintEffectSelectedPacket)
+              intoStream:kIsFacepaintEffectSelectedInputStream
+                   error:nil];
+}
+
+@end

mediapipe/examples/ios/faceeffect/Info.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>

mediapipe/examples/ios/faceeffect/main.m

@@ -0,0 +1,22 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char* argv[]) {
+  @autoreleasepool {
+    return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}

mediapipe/examples/python/upper_body_pose_tracker.py

@@ -150,7 +150,7 @@ class UpperBodyPoseTracker:
       success, input_frame = cap.read()
       if not success:
         break
-      input_frame = cv2.cvtColor(input_frame, cv2.COLOR_BGR2RGB)
+      input_frame = cv2.cvtColor(cv2.flip(input_frame, 1), cv2.COLOR_BGR2RGB)
       input_frame.flags.writeable = False
       _, output_frame = self._run_graph(input_frame)
       cv2.imshow('MediaPipe upper body pose tracker',

mediapipe/framework/BUILD

@@ -947,6 +947,7 @@ cc_library(
         ],
     }),
     visibility = [
+        "//mediapipe/calculators:__subpackages__",
         "//mediapipe/framework:__subpackages__",
         "//mediapipe/framework/port:__pkg__",
         "//mediapipe/util:__subpackages__",
@@ -1106,6 +1107,19 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "basic_types_registration",
+    srcs = ["basic_types_registration.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":type_map",
+        "//mediapipe/framework/port:integral_types",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "validated_graph_config",
     srcs = ["validated_graph_config.cc"],

mediapipe/framework/basic_types_registration.cc

@@ -0,0 +1,28 @@
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/type_map.h"
+
+#define MEDIAPIPE_REGISTER_GENERIC_TYPE(type)                                \
+  MEDIAPIPE_REGISTER_TYPE(                                                   \
+      ::mediapipe::type_map_internal::ReflectType<void(type*)>::Type, #type, \
+      nullptr, nullptr)
+
+// Note: we cannot define a type which type hash id is already in the map.
+// E.g. if tool::GetTypeHash<int>() == tool::GetTypeHash<int32>(), then only one
+// can be registered.
+
+MEDIAPIPE_REGISTER_GENERIC_TYPE(bool);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(double);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(float);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(int);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(int64);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(uint64);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<bool>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<double>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<float>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<int>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<int64>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<std::string>);
+MEDIAPIPE_REGISTER_GENERIC_TYPE(::std::vector<::std::vector<float>>);

mediapipe/framework/calculator_graph.cc

@@ -330,6 +330,9 @@ CalculatorGraph::~CalculatorGraph() {
 ::mediapipe::Status CalculatorGraph::InitializeDefaultExecutor(
     const ThreadPoolExecutorOptions* default_executor_options,
     bool use_application_thread) {
+#ifdef __EMSCRIPTEN__
+  use_application_thread = true;
+#endif  // __EMSCRIPTEN__
   // If specified, run synchronously on the calling thread.
   if (use_application_thread) {
     use_application_thread_ = true;

mediapipe/framework/collection_test.cc

@@ -218,10 +218,11 @@ template <typename T>
     EXPECT_EQ(values[5], collection_ptr->Get("TAG_C", 0));
 
     // Test const-ness.
-    EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
-                         collection.Get("TAG_A", 0))>::type>::value);
-    EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
-                        collection_ptr->Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(false, std::is_const<typename std::remove_reference<
+                         decltype(collection.Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(true,
+              std::is_const<typename std::remove_reference<
+                  decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
 
     // Test access using a range based for.
     int i = 0;
@@ -278,10 +279,11 @@ template <typename T>
     EXPECT_EQ(values[5], collection_ptr->Get("TAG_C", 0));
 
     // Test const-ness.
-    EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
-                         collection.Get("TAG_A", 0))>::type>::value);
-    EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
-                        collection_ptr->Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(false, std::is_const<typename std::remove_reference<
+                         decltype(collection.Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(true,
+              std::is_const<typename std::remove_reference<
+                  decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
 
     // Test access using a range based for.
     int i = 0;
@@ -309,10 +311,10 @@ template <typename T>
       // storage == kStoreValue.
       EXPECT_EQ(&values[i], collection_ptr->GetPtr(id));
       EXPECT_EQ(values[i], *collection_ptr->GetPtr(id));
-      EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
-                           *collection.GetPtr(id))>::type>::value);
-      EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
-                          *collection_ptr->GetPtr(id))>::type>::value);
+      EXPECT_EQ(false, std::is_const<typename std::remove_reference<
+                           decltype(*collection.GetPtr(id))>::type>::value);
+      EXPECT_EQ(true, std::is_const<typename std::remove_reference<
+                          decltype(*collection_ptr->GetPtr(id))>::type>::value);
       ++i;
     }
 
@@ -386,10 +388,11 @@ template <typename T>
     EXPECT_EQ(&values[5], collection_ptr->Get("TAG_C", 0));
 
     // Test const-ness.
-    EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
-                         collection.Get("TAG_A", 0))>::type>::value);
-    EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
-                        collection_ptr->Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(false, std::is_const<typename std::remove_reference<
+                         decltype(collection.Get("TAG_A", 0))>::type>::value);
+    EXPECT_EQ(true,
+              std::is_const<typename std::remove_reference<
+                  decltype(collection_ptr->Get("TAG_A", 0))>::type>::value);
 
     // Test access using a range based for.
     int i = 0;
@@ -473,8 +476,8 @@ TEST(CollectionTest, TestIteratorFunctions) {
   collection.GetPtr(collection.GetId("TAG_B", 1)) = &values[4];
   collection.GetPtr(collection.GetId("TAG_C", 0)) = &values[5];
 
-  EXPECT_EQ(false, std::is_const<typename std::remove_reference<decltype(
-                       collection.begin())>::type>::value);
+  EXPECT_EQ(false, std::is_const<typename std::remove_reference<
+                       decltype(collection.begin())>::type>::value);
   EXPECT_EQ(values[0], *collection.begin());
   EXPECT_EQ(false, collection.begin()->empty());
   EXPECT_EQ(false, (*collection.begin()).empty());
@@ -483,8 +486,8 @@ TEST(CollectionTest, TestIteratorFunctions) {
 
   const auto* collection_ptr = &collection;
 
-  EXPECT_EQ(true, std::is_const<typename std::remove_reference<decltype(
-                      *collection_ptr->begin())>::type>::value);
+  EXPECT_EQ(true, std::is_const<typename std::remove_reference<
+                      decltype(*collection_ptr->begin())>::type>::value);
   EXPECT_EQ(values[0], *collection_ptr->begin());
   EXPECT_EQ(false, collection_ptr->begin()->empty());
   EXPECT_EQ(false, (*collection_ptr->begin()).empty());

mediapipe/framework/deps/file_helpers.cc

@@ -16,6 +16,7 @@
 
 #ifdef _WIN32
 #include <Windows.h>
+#include <direct.h>
 #else
 #include <dirent.h>
 #endif  // _WIN32
@@ -27,7 +28,9 @@
 
 #include "mediapipe/framework/deps/canonical_errors.h"
 #include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/deps/status.h"
 #include "mediapipe/framework/deps/status_builder.h"
+#include "mediapipe/framework/deps/status_macros.h"
 
 namespace mediapipe {
 namespace file {
@@ -212,7 +215,7 @@ class DirectoryListing {
 ::mediapipe::Status Exists(absl::string_view file_name) {
   struct stat buffer;
   int status;
-  status = stat(file_name.data(), &buffer);
+  status = stat(std::string(file_name).c_str(), &buffer);
   if (status == 0) {
     return ::mediapipe::OkStatus();
   }
@@ -224,5 +227,30 @@ class DirectoryListing {
   }
 }
 
+#ifndef _WIN32
+int mkdir(std::string path) {
+  return ::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+}
+#else
+int mkdir(std::string path) { return _mkdir(path.c_str()); }
+#endif
+
+::mediapipe::Status RecursivelyCreateDir(absl::string_view path) {
+  if (path.empty() || Exists(path).ok()) {
+    return mediapipe::OkStatus();
+  }
+  auto split_path = file::SplitPath(path);
+  MP_RETURN_IF_ERROR(RecursivelyCreateDir(split_path.first));
+  if (mkdir(std::string(path)) != 0) {
+    switch (errno) {
+      case EACCES:
+        return ::mediapipe::PermissionDeniedError("Insufficient permissions.");
+      default:
+        return ::mediapipe::UnavailableError("Failed to create directory.");
+    }
+  }
+  return mediapipe::OkStatus();
+}
+
 }  // namespace file
 }  // namespace mediapipe

mediapipe/framework/deps/file_helpers.h

@@ -36,6 +36,8 @@ namespace file {
 
 ::mediapipe::Status Exists(absl::string_view file_name);
 
+::mediapipe::Status RecursivelyCreateDir(absl::string_view path);
+
 }  // namespace file
 }  // namespace mediapipe
 

mediapipe/framework/deps/threadpool_pthread_impl.cc

@@ -47,7 +47,8 @@ class ThreadPool::WorkerThread {
 ThreadPool::WorkerThread::WorkerThread(ThreadPool* pool,
                                        const std::string& name_prefix)
     : pool_(pool), name_prefix_(name_prefix) {
-  pthread_create(&thread_, nullptr, ThreadBody, this);
+  int res = pthread_create(&thread_, nullptr, ThreadBody, this);
+  CHECK_EQ(res, 0) << "pthread_create failed";
 }
 
 ThreadPool::WorkerThread::~WorkerThread() {}
@@ -59,9 +60,9 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
   int nice_priority_level =
       thread->pool_->thread_options().nice_priority_level();
   const std::set<int> selected_cpus = thread->pool_->thread_options().cpu_set();
+#if defined(__linux__)
   const std::string name =
       internal::CreateThreadName(thread->name_prefix_, syscall(SYS_gettid));
-#if defined(__linux__)
   if (nice_priority_level != 0) {
     if (nice(nice_priority_level) != -1 || errno == 0) {
       VLOG(1) << "Changed the nice priority level by " << nice_priority_level;
@@ -94,16 +95,19 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
                << "Failed to set name for thread: " << name;
   }
 #else
+  const std::string name = internal::CreateThreadName(thread->name_prefix_, 0);
   if (nice_priority_level != 0 || !selected_cpus.empty()) {
     LOG(ERROR) << "Thread priority and processor affinity feature aren't "
                   "supported on the current platform.";
   }
+#if __APPLE__
   int error = pthread_setname_np(name.c_str());
   if (error != 0) {
     LOG(ERROR) << "Error : " << strerror(error) << std::endl
                << "Failed to set name for thread: " << name;
   }
-#endif
+#endif  // __APPLE__
+#endif  // __linux__
   thread->pool_->RunWorker();
   return nullptr;
 }
@@ -178,6 +182,12 @@ const ThreadOptions& ThreadPool::thread_options() const {
 
 namespace internal {
 
+// TODO: revise this:
+// - thread_id is not portable
+// - the 16-byte limit is Linux-specific
+// - the std::thread implementation has a copy of this but doesn't use it
+// - why do we even need the thread id in the name? any thread list should show
+//   the id too.
 std::string CreateThreadName(const std::string& prefix, int thread_id) {
   std::string name = absl::StrCat(prefix, "/", thread_id);
   // 16 is the limit allowed by `pthread_setname_np`, including

mediapipe/framework/encode_binary_proto.bzl

@@ -1,3 +1,17 @@
+# Copyright 2019-2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """A rule for encoding a text format protocol buffer into binary.
 
 Example usage:

mediapipe/framework/formats/BUILD

@@ -14,6 +14,7 @@
 #
 
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+load("//mediapipe/framework:mediapipe_register_type.bzl", "mediapipe_register_type")
 
 package(
     default_visibility = ["//visibility:private"],
@@ -22,8 +23,6 @@ package(
 
 licenses(["notice"])
 
-exports_files(["LICENSE"])
-
 mediapipe_proto_library(
     name = "detection_proto",
     srcs = ["detection.proto"],
@@ -31,6 +30,18 @@ mediapipe_proto_library(
     deps = ["//mediapipe/framework/formats:location_data_proto"],
 )
 
+mediapipe_register_type(
+    base_name = "detection",
+    include_headers = ["mediapipe/framework/formats/detection.pb.h"],
+    types = [
+        "::mediapipe::Detection",
+        "::mediapipe::DetectionList",
+        "::std::vector<::mediapipe::Detection>",
+        "::std::vector<::mediapipe::DetectionList>",
+    ],
+    deps = ["//mediapipe/framework/formats:detection_cc_proto"],
+)
+
 mediapipe_proto_library(
     name = "classification_proto",
     srcs = ["classification.proto"],
@@ -213,6 +224,18 @@ mediapipe_proto_library(
     deps = ["//mediapipe/framework/formats:location_data_proto"],
 )
 
+mediapipe_register_type(
+    base_name = "rect",
+    include_headers = ["mediapipe/framework/formats/rect.pb.h"],
+    types = [
+        "::mediapipe::Rect",
+        "::mediapipe::NormalizedRect",
+        "::std::vector<::mediapipe::Rect>",
+        "::std::vector<::mediapipe::NormalizedRect>",
+    ],
+    deps = [":rect_cc_proto"],
+)
+
 mediapipe_proto_library(
     name = "landmark_proto",
     srcs = ["landmark.proto"],
@@ -221,9 +244,48 @@ mediapipe_proto_library(
     ],
 )
 
+mediapipe_register_type(
+    base_name = "landmark",
+    include_headers = ["mediapipe/framework/formats/landmark.pb.h"],
+    types = [
+        "::mediapipe::Landmark",
+        "::mediapipe::NormalizedLandmark",
+        "::mediapipe::NormalizedLandmarkList",
+        "::std::vector<::mediapipe::Landmark>",
+        "::std::vector<::mediapipe::NormalizedLandmark>",
+        "::std::vector<::mediapipe::NormalizedLandmarkList>",
+    ],
+    deps = [":landmark_cc_proto"],
+)
+
 # Expose the proto source files for building mediapipe AAR.
 filegroup(
     name = "protos_src",
     srcs = glob(["*.proto"]),
     visibility = ["//mediapipe:__subpackages__"],
 )
+
+cc_library(
+    name = "image_frame_pool",
+    srcs = ["image_frame_pool.cc"],
+    hdrs = ["image_frame_pool.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":image_frame",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
+cc_test(
+    name = "image_frame_pool_test",
+    size = "small",
+    srcs = ["image_frame_pool_test.cc"],
+    tags = ["linux"],
+    deps = [
+        ":image_frame_pool",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)

mediapipe/framework/formats/annotation/BUILD

@@ -20,8 +20,6 @@ package(default_visibility = ["//visibility:private"])
 
 licenses(["notice"])
 
-exports_files(["LICENSE"])
-
 mediapipe_proto_library(
     name = "locus_proto",
     srcs = ["locus.proto"],

mediapipe/framework/formats/image_frame_pool.cc

@@ -0,0 +1,88 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/image_frame_pool.h"
+
+#include "absl/synchronization/mutex.h"
+
+namespace mediapipe {
+
+ImageFramePool::ImageFramePool(int width, int height,
+                               ImageFormat::Format format, int keep_count)
+    : width_(width),
+      height_(height),
+      format_(format),
+      keep_count_(keep_count) {}
+
+ImageFrameSharedPtr ImageFramePool::GetBuffer() {
+  std::unique_ptr<ImageFrame> buffer;
+
+  {
+    absl::MutexLock lock(&mutex_);
+    if (available_.empty()) {
+      // Fix alignment at 4 for best compatability with OpenGL.
+      buffer = std::make_unique<ImageFrame>(
+          format_, width_, height_, ImageFrame::kGlDefaultAlignmentBoundary);
+      if (!buffer) return nullptr;
+    } else {
+      buffer = std::move(available_.back());
+      available_.pop_back();
+    }
+
+    ++in_use_count_;
+  }
+
+  // Return a shared_ptr with a custom deleter that adds the buffer back
+  // to our available list.
+  std::weak_ptr<ImageFramePool> weak_pool(shared_from_this());
+  return std::shared_ptr<ImageFrame>(buffer.release(),
+                                     [weak_pool](ImageFrame* buf) {
+                                       auto pool = weak_pool.lock();
+                                       if (pool) {
+                                         pool->Return(buf);
+                                       } else {
+                                         delete buf;
+                                       }
+                                     });
+}
+
+std::pair<int, int> ImageFramePool::GetInUseAndAvailableCounts() {
+  absl::MutexLock lock(&mutex_);
+  return {in_use_count_, available_.size()};
+}
+
+void ImageFramePool::Return(ImageFrame* buf) {
+  std::vector<std::unique_ptr<ImageFrame>> trimmed;
+  {
+    absl::MutexLock lock(&mutex_);
+    --in_use_count_;
+    available_.emplace_back(buf);
+    TrimAvailable(&trimmed);
+  }
+  // The trimmed buffers will be released without holding the lock.
+}
+
+void ImageFramePool::TrimAvailable(
+    std::vector<std::unique_ptr<ImageFrame>>* trimmed) {
+  int keep = std::max(keep_count_ - in_use_count_, 0);
+  if (available_.size() > keep) {
+    auto trim_it = std::next(available_.begin(), keep);
+    if (trimmed) {
+      std::move(trim_it, available_.end(), std::back_inserter(*trimmed));
+    }
+    available_.erase(trim_it, available_.end());
+  }
+}
+
+}  // namespace mediapipe

mediapipe/framework/formats/image_frame_pool.h

@@ -0,0 +1,78 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Consider this file an implementation detail. None of this is part of the
+// public API.
+
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
+
+#include <utility>
+#include <vector>
+
+#include "absl/synchronization/mutex.h"
+#include "mediapipe/framework/formats/image_frame.h"
+
+namespace mediapipe {
+
+using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
+
+class ImageFramePool : public std::enable_shared_from_this<ImageFramePool> {
+ public:
+  // Creates a pool. This pool will manage buffers of the specified dimensions,
+  // and will keep keep_count buffers around for reuse.
+  // We enforce creation as a shared_ptr so that we can use a weak reference in
+  // the buffers' deleters.
+  static std::shared_ptr<ImageFramePool> Create(int width, int height,
+                                                ImageFormat::Format format,
+                                                int keep_count) {
+    return std::shared_ptr<ImageFramePool>(
+        new ImageFramePool(width, height, format, keep_count));
+  }
+
+  // Obtains a buffers. May either be reused or created anew.
+  ImageFrameSharedPtr GetBuffer();
+
+  int width() const { return width_; }
+  int height() const { return height_; }
+  ImageFormat::Format format() const { return format_; }
+
+  // This method is meant for testing.
+  std::pair<int, int> GetInUseAndAvailableCounts();
+
+ private:
+  ImageFramePool(int width, int height, ImageFormat::Format format,
+                 int keep_count);
+
+  // Return a buffer to the pool.
+  void Return(ImageFrame* buf);
+
+  // If the total number of buffers is greater than keep_count, destroys any
+  // surplus buffers that are no longer in use.
+  void TrimAvailable(std::vector<std::unique_ptr<ImageFrame>>* trimmed)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+  const int width_;
+  const int height_;
+  const ImageFormat::Format format_;
+  const int keep_count_;
+
+  absl::Mutex mutex_;
+  int in_use_count_ ABSL_GUARDED_BY(mutex_) = 0;
+  std::vector<std::unique_ptr<ImageFrame>> available_ ABSL_GUARDED_BY(mutex_);
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_

mediapipe/framework/formats/image_frame_pool_test.cc

@@ -0,0 +1,103 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/image_frame_pool.h"
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace {
+
+using Pair = std::pair<int, int>;
+
+constexpr int kWidth = 300;
+constexpr int kHeight = 200;
+constexpr ImageFormat::Format kFormat = ImageFormat::SRGBA;
+constexpr int kKeepCount = 2;
+
+class ImageFramePoolTest : public ::testing::Test {
+ protected:
+  ImageFramePoolTest() {
+    pool_ = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
+  }
+
+  void SetUp() override {}
+
+  std::shared_ptr<ImageFramePool> pool_;
+};
+
+TEST_F(ImageFramePoolTest, GetBuffer) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  auto buffer = pool_->GetBuffer();
+  EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
+  buffer = nullptr;
+  EXPECT_EQ(Pair(0, 1), pool_->GetInUseAndAvailableCounts());
+  buffer = pool_->GetBuffer();
+  EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST_F(ImageFramePoolTest, GetMoreBuffers) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  std::vector<ImageFrameSharedPtr> buffers;
+
+  // Create kKeepCount + 1 buffers
+  for (int i = 0; i <= kKeepCount; i++) {
+    buffers.emplace_back(pool_->GetBuffer());
+  }
+  EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete one
+  buffers.resize(kKeepCount);
+  EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete all
+  buffers.resize(0);
+  EXPECT_EQ(Pair(0, kKeepCount), pool_->GetInUseAndAvailableCounts());
+
+  // Create one more
+  buffers.emplace_back(pool_->GetBuffer());
+  EXPECT_EQ(Pair(1, kKeepCount - 1), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST_F(ImageFramePoolTest, DeleteNotLast) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  std::vector<ImageFrameSharedPtr> buffers;
+
+  // Create kKeepCount + 1 buffers
+  for (int i = 0; i <= kKeepCount; i++) {
+    buffers.emplace_back(pool_->GetBuffer());
+  }
+  EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete second
+  buffers.erase(buffers.begin() + 1);
+  EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete first
+  buffers.erase(buffers.begin());
+  EXPECT_EQ(Pair(kKeepCount - 1, 1), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST(ImageFrameBufferPoolStaticTest, BufferCanOutlivePool) {
+  auto pool = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
+  auto buffer = pool->GetBuffer();
+  pool = nullptr;
+  buffer = nullptr;
+}
+
+}  // anonymous namespace
+}  // namespace mediapipe

mediapipe/framework/formats/landmark.proto

@@ -28,9 +28,17 @@ message Landmark {
   optional float z = 3;
 
   // Landmark visibility. Float score of whether landmark is visible or occluded
-  // by other obects. Depending on the model, visibility value is either a
+  // by other objects. Landmark considered as invisible also if it is
+  // not present on the screen (out of scene bounds).
+  // Depending on the model, visibility value is either a
   // sigmoid or an argument of sigmoid.
   optional float visibility = 4;
+
+  // Landmark presence. Float score of whether landmark is present on the scene
+  // (located within scene bounds).
+  // Depending on the model, presence value is either a result of sigmoid
+  // or an argument of sigmoid function to get landmark presence probability.
+  optional float presence = 5;
 }
 
 // Group of Landmark protos.
@@ -45,6 +53,7 @@ message NormalizedLandmark {
   optional float y = 2;
   optional float z = 3;
   optional float visibility = 4;
+  optional float presence = 5;
 }
 
 // Group of NormalizedLandmark protos.

mediapipe/framework/formats/motion/BUILD

@@ -18,8 +18,6 @@
 
 licenses(["notice"])
 
-exports_files(["LICENSE"])
-
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 
 package(default_visibility = ["//visibility:private"])

mediapipe/framework/input_stream_manager.cc

@@ -163,9 +163,10 @@ template <typename Container>
     }
     queue_became_full = (!was_queue_full && max_queue_size_ != -1 &&
                          queue_.size() >= max_queue_size_);
-    VLOG_IF(3, queue_.size() > 1)
-        << "Queue size greater than 1: stream name: " << name_
-        << " queue_size: " << queue_.size();
+    if (queue_.size() > 1) {
+      VLOG(3) << "Queue size greater than 1: stream name: " << name_
+              << " queue_size: " << queue_.size();
+    }
     VLOG(3) << "Input stream:" << name_
             << " becomes non-empty status:" << queue_became_non_empty
             << " Size: " << queue_.size();

mediapipe/framework/legacy_calculator_support.h

@@ -66,6 +66,7 @@ class LegacyCalculatorSupport {
   };
 };
 
+#if !defined(_MSC_VER)
 // We only declare this variable for two specializations of the template because
 // it is only meant to be used for these two types.
 // Note that, since these variables are members of specific template
@@ -79,6 +80,7 @@ thread_local CalculatorContext*
 template <>
 thread_local CalculatorContract*
     LegacyCalculatorSupport::Scoped<CalculatorContract>::current_;
+#endif
 
 }  // namespace mediapipe
 

mediapipe/framework/mediapipe_register_type.bzl

@@ -0,0 +1,117 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A rule for registering types with mediapipe.
+
+Example usage:
+    mediapipe_proto_library(
+        name = "foo_proto",
+        srcs = ["foo.proto"],
+    )
+
+    load("//mediapipe/framework:mediapipe_register_type.bzl",
+         "mediapipe_register_type")
+
+    # Creates rules "foo_registration"
+    mediapipe_register_type(
+        base_name = "foo"
+        include_headers = ["mediapipe/framework/formats/foo.proto.h""],
+        types = [
+            "::mediapipe::Foo",
+            "::mediapipe::FooList",
+            "::std::vector<::mediapipe::Foo>",
+        ],
+        deps = [":foo_cc_proto"],
+    )
+
+Args
+  base_name: The base name of the target
+             (name + "_registration" will be created).
+  types: A list of C++ classes to register using MEDIAPIPE_REGISTER_TYPE.
+  deps: A list of cc deps.
+  include_headers: A list of header files that must be included.
+"""
+
+load("//mediapipe/framework/tool:build_defs.bzl", "clean_dep")
+
+def _mediapipe_register_type_generate_cc_impl(ctx):
+    """Generate a cc file that registers types with mediapipe."""
+    file_data_template = '''
+{include_headers}
+#include "mediapipe/framework/type_map.h"
+
+{registration_commands}
+'''
+    header_lines = []
+    for header in ctx.attr.include_headers:
+        header_lines.append('#include "{}"'.format(header))
+    registration_lines = []
+    for registration_type in ctx.attr.types:
+        if " " in registration_type:
+            fail(('registration type "{}" should be fully qualified ' +
+                  "and must not include spaces").format(registration_type))
+        registration_lines.append(
+            "#define TEMP_MP_TYPE {}".format(registration_type),
+        )
+        registration_lines.append(
+            ("MEDIAPIPE_REGISTER_TYPE(\n" +
+             "    TEMP_MP_TYPE,\n" +
+             '    "{}",\n'.format(registration_type) +
+             "    nullptr, nullptr);\n"),
+        )
+        registration_lines.append("#undef TEMP_MP_TYPE")
+
+    file_data = file_data_template.format(
+        include_headers = "\n".join(header_lines),
+        registration_commands = "\n".join(registration_lines),
+    )
+
+    ctx.actions.write(ctx.outputs.output, file_data)
+
+mediapipe_register_type_generate_cc = rule(
+    implementation = _mediapipe_register_type_generate_cc_impl,
+    attrs = {
+        "deps": attr.label_list(),
+        "types": attr.string_list(
+            mandatory = True,
+        ),
+        "include_headers": attr.string_list(
+            mandatory = True,
+        ),
+        "output": attr.output(),
+    },
+)
+
+def mediapipe_register_type(
+        base_name,
+        types,
+        deps = [],
+        include_headers = [],
+        visibility = ["//visibility:public"]):
+    mediapipe_register_type_generate_cc(
+        name = base_name + "_registration_cc",
+        types = types,
+        include_headers = include_headers,
+        output = base_name + "_registration.cc",
+    )
+
+    native.cc_library(
+        name = base_name + "_registration",
+        srcs = [base_name + "_registration.cc"],
+        deps = depset(deps + [
+            clean_dep("//mediapipe/framework:type_map"),
+        ]),
+        visibility = visibility,
+        alwayslink = 1,
+    )

mediapipe/framework/packet.h

@@ -439,13 +439,18 @@ struct is_concrete_proto_t
 template <typename T>
 struct MessageRegistrationImpl {
   static NoDestructor<mediapipe::RegistrationToken> registration;
+  // This could have been a lambda inside registration's initializer below, but
+  // MSVC has a bug with lambdas, so we put it here as a workaround.
+  static std::unique_ptr<Holder<T>> CreateMessageHolder() {
+    return absl::make_unique<Holder<T>>(new T);
+  }
 };
 
 // Static members of template classes can be defined in the header.
 template <typename T>
 NoDestructor<mediapipe::RegistrationToken>
     MessageRegistrationImpl<T>::registration(MessageHolderRegistry::Register(
-        T{}.GetTypeName(), [] { return absl::make_unique<Holder<T>>(new T); }));
+        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder));
 
 // For non-Message payloads, this does nothing.
 template <typename T, typename Enable = void>

mediapipe/framework/port.h

@@ -52,12 +52,12 @@
 
 // Compile time target platform definitions.
 // Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
-#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0
 #define MEDIAPIPE_OPENGL_ES_20 200
+#define MEDIAPIPE_OPENGL_ES_30 300
 #define MEDIAPIPE_OPENGL_ES_31 310
 
 #if defined(MEDIAPIPE_DISABLE_GPU)
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_OPENGL_ES_VERSION 0
 #define MEDIAPIPE_METAL_ENABLED 0
 #else
 #if defined(MEDIAPIPE_ANDROID)
@@ -71,11 +71,11 @@
 #define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
 #define MEDIAPIPE_METAL_ENABLED 1
 #elif defined(MEDIAPIPE_OSX)
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_OPENGL_ES_VERSION 0
 #define MEDIAPIPE_METAL_ENABLED 1
 #else
-// GPU is not supported on Linux yet.
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+// WebGL config.
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_30
 #define MEDIAPIPE_METAL_ENABLED 0
 #endif
 #endif

mediapipe/framework/profiler/BUILD

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 licenses(["notice"])
 
@@ -269,7 +268,7 @@ cc_test(
 
 cc_library(
     name = "profiler_resource_util",
-    srcs = select({
+    srcs = ["profiler_resource_util_common.cc"] + select({
         "//conditions:default": ["profiler_resource_util.cc"],
         "//mediapipe:android": ["profiler_resource_util_android.cc"],
         "//mediapipe:ios": ["profiler_resource_util_ios.cc"],
@@ -288,15 +287,22 @@ cc_library(
     deps = [
         "@com_google_absl//absl/strings",
         "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:statusor",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/flags:flag",
+        "//mediapipe/framework/deps:file_path",
     ] + select({
         "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
         ],
         "//mediapipe:android": [
             "//mediapipe/java/com/google/mediapipe/framework/jni:jni_util",
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:apple": [
+            "//mediapipe/framework/port:file_helpers",
         ],
-        "//mediapipe:apple": [],
     }),
 )