Project import generated by Copybara.

GitOrigin-RevId: 89ada20661cae165e3258e6057fd507e3aae7f48

.bazelrc

@@ -1,20 +1,30 @@
 # The bazelrc file for MediaPipe OSS.
 
+# Tensorflow needs remote repo
+common --experimental_repo_remote_exec
+
 # Basic build settings
 build --jobs 128
 build --define='absl=1'
-build --cxxopt='-std=c++14'
+build --enable_platform_specific_config
-build --copt='-Wno-sign-compare'
-build --copt='-Wno-unused-function'
-build --copt='-Wno-uninitialized'
-build --copt='-Wno-unused-result'
-build --copt='-Wno-comment'
-build --copt='-Wno-return-type'
-build --copt='-Wno-unused-local-typedefs'
-build --copt='-Wno-ignored-attributes'
 
-# Tensorflow needs remote repo
+# Linux
-build --experimental_repo_remote_exec
+build:linux --cxxopt=-std=c++14
+build:linux --host_cxxopt=-std=c++14
+build:linux --copt=-w
+
+# windows
+build:windows --cxxopt=/std:c++14
+build:windows --host_cxxopt=/std:c++14
+build:windows --copt=/w
+# For using M_* math constants on Windows with MSVC.
+build:windows --copt=/D_USE_MATH_DEFINES
+build:windows --host_copt=/D_USE_MATH_DEFINES
+
+# macOS
+build:macos --cxxopt=-std=c++14
+build:macos --host_cxxopt=-std=c++14
+build:macos --copt=-w
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
@@ -69,3 +79,7 @@ build:ios_arm64e --watchos_cpus=armv7k
 build:ios_fat --config=ios
 build:ios_fat --ios_multi_cpus=armv7,arm64
 build:ios_fat --watchos_cpus=armv7k
+
+build:darwin_x86_64 --apple_platform_type=macos
+build:darwin_x86_64 --macos_minimum_os=10.12
+build:darwin_x86_64 --cpu=darwin_x86_64

Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:latest
+FROM ubuntu:18.04
 
 MAINTAINER <[email protected]>
 
@@ -25,11 +25,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         ca-certificates \
         curl \
+        ffmpeg \
         git \
         wget \
         unzip \
-        python \
+        python3-dev \
-        python-pip \
+        python3-opencv \
         python3-pip \
         libopencv-core-dev \
         libopencv-highgui-dev \
@@ -43,9 +44,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade setuptools
+RUN pip3 install --upgrade setuptools
-RUN pip install future
+RUN pip3 install wheel
-RUN pip3 install six
+RUN pip3 install future
+RUN pip3 install six==1.14.0
+RUN pip3 install tensorflow==1.14.0
+RUN pip3 install tf_slim
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
 ARG BAZEL_VERSION=2.0.0

README.md

@@ -1,36 +1,38 @@
 ![MediaPipe](mediapipe/docs/images/mediapipe_small.png?raw=true "MediaPipe logo")
 =======================================================================
 
-[MediaPipe](http://mediapipe.dev) is a framework for building multimodal (eg. video, audio, any time series data), cross platform (i.e Android, iOS, web, edge devices) applied ML pipelines. With MediaPipe, a perception pipeline can be built as a graph of modular components, including, for instance, inference models (e.g., TensorFlow, TFLite) and media processing functions.
+[MediaPipe](http://mediapipe.dev) is the simplest way for researchers and developers to build world-class ML solutions and applications for mobile, edge, cloud and the web.
-
-![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)
-
-> "<em>MediaPipe has made it extremely easy to build our 3D person pose reconstruction demo app, facilitating accelerated neural network inference on device and synchronization of our result visualization with the video capture stream. Highly recommended!</em>" - George Papandreou, CTO, [Ariel AI](https://arielai.com)
 
 ## ML Solutions in MediaPipe
 
-* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Mesh](mediapipe/docs/face_mesh_mobile_gpu.md)
+* [Hand Detection](mediapipe/docs/hand_detection_mobile_gpu.md)
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 * [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
-* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
-* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 * [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
 * [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
-* [AutoFlip](mediapipe/docs/autoflip.md)
+* [AutoFlip: Intelligent Video Reframing](mediapipe/docs/autoflip.md)
+* [KNIFT: Template Matching with Neural Image Features](mediapipe/docs/template_matching_mobile_cpu.md)
 
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
-![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
+![face_mesh](mediapipe/docs/images/mobile/face_mesh_android_gpu_small.gif)
-![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
+![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
 ![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
+![objectron_shoes](mediapipe/docs/images/mobile/objectron_shoe_android_gpu_small.gif)
+![objectron_chair](mediapipe/docs/images/mobile/objectron_chair_android_gpu_small.gif)
+![template_matching](mediapipe/docs/images/mobile/template_matching_android_cpu_small.gif)
 
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).
 
 ## Getting started
-See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
+See mobile, desktop, web and Google Coral [examples](mediapipe/docs/examples.md).
-
-Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 
 ## Documentation
 [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
@@ -40,10 +42,15 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 ## Visualizing MediaPipe graphs
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
+## Google Open Source Code search
+Search MediaPipe Github repository using [Google Open Source code search](https://t.co/LSZnbMUUnT?amp=1)
+
 ## Videos
 *  [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
 
 ## Publications
+* [MediaPipe KNIFT: Template-based Feature Matching](https://mediapipe.page.link/knift-blog)
+* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
 * [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
 * [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)
 * [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
@@ -52,6 +59,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
+* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
 * [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
 * [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
 * [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
@@ -62,7 +70,9 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
 * [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
 
-## Community forum
+## Community
+*  [Awesome MediaPipe: curation of code related to MediaPipe](https://mediapipe.org)
+*  [Slack community for MediaPipe users](https://mediapipe.slack.com)
 *  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
 
 ## Alpha Disclaimer

WORKSPACE

@@ -54,17 +54,15 @@ http_archive(
 # gflags needed by glog
 http_archive(
     name = "com_github_gflags_gflags",
-    sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
+    strip_prefix = "gflags-2.2.2",
-    strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
+    sha256 = "19713a36c9f32b33df59d1c79b4958434cb005b5b47dc5400a7a4b078111d9b5",
-    urls = [
+    url = "https://github.com/gflags/gflags/archive/v2.2.2.zip",
-        "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
-        "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
-    ],
 )
 
-# glog
+# glog v0.3.5
+# TODO: Migrate MediaPipe to use com_github_glog_glog on all platforms.
 http_archive(
-    name = "com_github_glog_glog",
+    name = "com_github_glog_glog_v_0_3_5",
     url = "https://github.com/google/glog/archive/v0.3.5.zip",
     sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
     strip_prefix = "glog-0.3.5",
@@ -77,6 +75,16 @@ http_archive(
     ],
 )
 
+# 2020-02-16
+http_archive(
+    name = "com_github_glog_glog",
+    strip_prefix = "glog-3ba8976592274bc1f907c402ce22558011d6fc5e",
+    sha256 = "feca3c7e29a693cab7887409756d89d342d4a992d54d7c5599bebeae8f7b50be",
+    urls = [
+        "https://github.com/google/glog/archive/3ba8976592274bc1f907c402ce22558011d6fc5e.zip",
+    ],
+)
+
 # easyexif
 http_archive(
     name = "easyexif",
@@ -101,51 +109,30 @@ http_archive(
     urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
 )
 
+http_archive(
+    name = "com_google_protobuf",
+    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
+    strip_prefix = "protobuf-3.11.4",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
+    patches = [
+        "@//third_party:com_google_protobuf_fixes.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+)
+
 http_archive(
     name = "com_google_audio_tools",
     strip_prefix = "multichannel-audio-tools-master",
     urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
 )
 
-# Needed by TensorFlow
-http_archive(
-    name = "io_bazel_rules_closure",
-    sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
-    strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
-    urls = [
-        "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",  # 2019-04-04
-    ],
-)
-
-# 2020-04-01
-_TENSORFLOW_GIT_COMMIT = "805e47cea96c7e8c6fccf494d40a2392dc99fdd8"
-_TENSORFLOW_SHA256= "9ee3ae604c2e1345ac60345becee6d659364721513f9cb8652eb2e7138320ca5"
-http_archive(
-    name = "org_tensorflow",
-    urls = [
-      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
-      "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
-    ],
-    patches = [
-        "@//third_party:org_tensorflow_compatibility_fixes.diff",
-        "@//third_party:org_tensorflow_protobuf_updates.diff",
-    ],
-    patch_args = [
-        "-p1",
-    ],
-    strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
-    sha256 = _TENSORFLOW_SHA256,
-)
-
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace(tf_repo_name = "org_tensorflow")
-
 http_archive(
     name = "ceres_solver",
     url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
     patches = [
-        "@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
+        "@//third_party:ceres_solver_compatibility_fixes.diff"
     ],
     patch_args = [
         "-p1",
@@ -178,6 +165,12 @@ new_local_repository(
     path = "/usr",
 )
 
+new_local_repository(
+    name = "windows_opencv",
+    build_file = "@//third_party:opencv_windows.BUILD",
+    path = "C:\\opencv\\build",
+)
+
 http_archive(
     name = "android_opencv",
     build_file = "@//third_party:opencv_android.BUILD",
@@ -236,6 +229,15 @@ load(
 
 swift_rules_dependencies()
 
+http_archive(
+    name = "build_bazel_apple_support",
+    sha256 = "122ebf7fe7d1c8e938af6aeaee0efe788a3a2449ece5a8d6a428cb18d6f88033",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
+        "https://github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
+    ],
+)
+
 load(
     "@build_bazel_apple_support//lib:repositories.bzl",
     "apple_support_dependencies",
@@ -299,3 +301,37 @@ maven_install(
     fetch_sources = True,
     version_conflict_policy = "pinned",
 )
+
+# Needed by TensorFlow
+http_archive(
+    name = "io_bazel_rules_closure",
+    sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
+    strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
+    urls = [
+        "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",  # 2019-04-04
+    ],
+)
+
+#Tensorflow repo should always go after the other external dependencies.
+# 2020-05-11
+_TENSORFLOW_GIT_COMMIT = "7c09d15f9fcc14343343c247ebf5b8e0afe3e4aa"
+_TENSORFLOW_SHA256= "673d00cbd2676ae43df1993e0d28c10b5ffbe96d9e2ab29f88a77b43c0211299"
+http_archive(
+    name = "org_tensorflow",
+    urls = [
+      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+      "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+    ],
+    patches = [
+        "@//third_party:org_tensorflow_compatibility_fixes.diff",
+    ],
+    patch_args = [
+        "-p1",
+    ],
+    strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
+    sha256 = _TENSORFLOW_SHA256,
+)
+
+load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+tf_workspace(tf_repo_name = "org_tensorflow")

mediapipe/BUILD

@@ -134,6 +134,11 @@ config_setting(
     ]
 ]
 
+config_setting(
+    name = "windows",
+    values = {"cpu": "x64_windows"},
+)
+
 exports_files(
     ["provisioning_profile.mobileprovision"],
     visibility = ["//visibility:public"],

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -184,23 +184,14 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
   use_local_timestamp_ = spectrogram_options.use_local_timestamp();
 
   if (spectrogram_options.frame_duration_seconds() <= 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Invalid or missing frame_duration_seconds.\n"
-           "frame_duration_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
   }
   if (spectrogram_options.frame_overlap_seconds() >=
       spectrogram_options.frame_duration_seconds()) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Invalid frame_overlap_seconds.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds()
-        << "\nframe_duration_seconds: "
-        << spectrogram_options.frame_duration_seconds();
   }
   if (spectrogram_options.frame_overlap_seconds() < 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Frame_overlap_seconds is < 0.0.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
   }
 
   TimeSeriesHeader input_header;
@@ -212,9 +203,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   if (!spectrogram_options.allow_multichannel_input() &&
       num_input_channels_ != 1) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "The current setting only supports single-channel input. Please set "
-           "allow_multichannel_input.\n";
   }
 
   frame_duration_samples_ =
@@ -293,10 +282,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   const Matrix& input_stream = cc->Inputs().Index(0).Get<Matrix>();
   if (input_stream.rows() != num_input_channels_) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Number of input channels do not correspond to the number of rows "
-        << "in the input matrix: " << num_input_channels_ << "channels vs "
-        << input_stream.rows() << " rows";
   }
 
   cumulative_input_samples_ += input_stream.cols();

mediapipe/calculators/core/BUILD

@@ -500,6 +500,7 @@ cc_library(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:options_util",
     ],
     alwayslink = 1,
 )
@@ -815,6 +816,38 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "split_normalized_landmark_list_calculator",
+    srcs = ["split_normalized_landmark_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "split_normalized_landmark_list_calculator_test",
+    srcs = ["split_normalized_landmark_list_calculator_test.cc"],
+    deps = [
+        ":split_normalized_landmark_list_calculator",
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_library(
     name = "dequantize_byte_array_calculator",
     srcs = ["dequantize_byte_array_calculator.cc"],

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -51,8 +51,8 @@ namespace mediapipe {
 class ConstantSidePacketCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    const auto& options = cc->Options().GetExtension(
+    const auto& options =
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
                  options.packet_size())
         << "Number of output side packets has to be same as number of packets "
@@ -80,8 +80,8 @@ class ConstantSidePacketCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    const auto& options = cc->Options().GetExtension(
+    const auto& options =
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     int index = 0;
     for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
          id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {

mediapipe/calculators/core/packet_thinner_calculator.cc

@@ -24,11 +24,13 @@
 #include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/options_util.h"
 
 namespace mediapipe {
 
 namespace {
 const double kTimebaseUs = 1000000;  // Microseconds.
+const char* const kOptionsTag = "OPTIONS";
 const char* const kPeriodTag = "PERIOD";
 }  // namespace
 
@@ -63,9 +65,15 @@ const char* const kPeriodTag = "PERIOD";
 // Thinning period can be provided in the calculator options or via a
 // side packet with the tag "PERIOD".
 //
+// Calculator options provided optionally with the "OPTIONS" input
+// sidepacket tag will be merged with this calculator's node options, i.e.,
+// singular fields of the side packet will overwrite the options defined in the
+// node, and repeated fields will concatenate.
+//
 // Example config:
 // node {
 //   calculator: "PacketThinnerCalculator"
+//   input_side_packet: "OPTIONS:calculator_options"
 //   input_stream: "signal"
 //   output_stream: "output"
 //   options {
@@ -83,6 +91,9 @@ class PacketThinnerCalculator : public CalculatorBase {
   ~PacketThinnerCalculator() override {}
 
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    if (cc->InputSidePackets().HasTag(kOptionsTag)) {
+      cc->InputSidePackets().Tag(kOptionsTag).Set<CalculatorOptions>();
+    }
     cc->Inputs().Index(0).SetAny();
     cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
     if (cc->InputSidePackets().HasTag(kPeriodTag)) {
@@ -143,7 +154,9 @@ TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; }
 }  // namespace
 
 ::mediapipe::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
-  auto& options = cc->Options<PacketThinnerCalculatorOptions>();
+  PacketThinnerCalculatorOptions options = mediapipe::tool::RetrieveOptions(
+      cc->Options<PacketThinnerCalculatorOptions>(), cc->InputSidePackets(),
+      kOptionsTag);
 
   thinner_type_ = options.thinner_type();
   // This check enables us to assume only two thinner types exist in Process()

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -93,8 +93,7 @@ class PreviousLoopbackCalculator : public CalculatorBase {
         // MAIN packet, hence not caring about corresponding loop packet.
         loop_timestamp = Timestamp::Unset();
       }
-      main_packet_specs_.push_back({.timestamp = main_packet.Timestamp(),
+      main_packet_specs_.push_back({main_packet.Timestamp(), loop_timestamp});
-                                    .loop_timestamp = loop_timestamp});
       prev_main_ts_ = main_packet.Timestamp();
     }
 

mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc

@@ -0,0 +1,165 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+
+// Splits an input packet with NormalizedLandmarkList into
+// multiple NormalizedLandmarkList output packets using the [begin, end) ranges
+// specified in SplitVectorCalculatorOptions. If the option "element_only" is
+// set to true, all ranges should be of size 1 and all outputs will be elements
+// of type NormalizedLandmark. If "element_only" is false, ranges can be
+// non-zero in size and all outputs will be of type NormalizedLandmarkList.
+// If the option "combine_outputs" is set to true, only one output stream can be
+// specified and all ranges of elements will be combined into one
+// NormalizedLandmarkList.
+class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() != 0);
+
+    cc->Inputs().Index(0).Set<NormalizedLandmarkList>();
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    if (options.combine_outputs()) {
+      RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+      cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
+      for (int i = 0; i < options.ranges_size() - 1; ++i) {
+        for (int j = i + 1; j < options.ranges_size(); ++j) {
+          const auto& range_0 = options.ranges(i);
+          const auto& range_1 = options.ranges(j);
+          if ((range_0.begin() >= range_1.begin() &&
+               range_0.begin() < range_1.end()) ||
+              (range_1.begin() >= range_0.begin() &&
+               range_1.begin() < range_0.end())) {
+            return ::mediapipe::InvalidArgumentError(
+                "Ranges must be non-overlapping when using combine_outputs "
+                "option.");
+          }
+        }
+      }
+    } else {
+      if (cc->Outputs().NumEntries() != options.ranges_size()) {
+        return ::mediapipe::InvalidArgumentError(
+            "The number of output streams should match the number of ranges "
+            "specified in the CalculatorOptions.");
+      }
+
+      // Set the output types for each output stream.
+      for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+        if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+            options.ranges(i).begin() >= options.ranges(i).end()) {
+          return ::mediapipe::InvalidArgumentError(
+              "Indices should be non-negative and begin index should be less "
+              "than the end index.");
+        }
+        if (options.element_only()) {
+          if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+            return ::mediapipe::InvalidArgumentError(
+                "Since element_only is true, all ranges should be of size 1.");
+          }
+          cc->Outputs().Index(i).Set<NormalizedLandmark>();
+        } else {
+          cc->Outputs().Index(i).Set<NormalizedLandmarkList>();
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    element_only_ = options.element_only();
+    combine_outputs_ = options.combine_outputs();
+
+    for (const auto& range : options.ranges()) {
+      ranges_.push_back({range.begin(), range.end()});
+      max_range_end_ = std::max(max_range_end_, range.end());
+      total_elements_ += range.end() - range.begin();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const NormalizedLandmarkList& input =
+        cc->Inputs().Index(0).Get<NormalizedLandmarkList>();
+    RET_CHECK_GE(input.landmark_size(), max_range_end_);
+
+    if (combine_outputs_) {
+      NormalizedLandmarkList output;
+      for (int i = 0; i < ranges_.size(); ++i) {
+        for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+          const NormalizedLandmark& input_landmark = input.landmark(j);
+          *output.add_landmark() = input_landmark;
+        }
+      }
+      RET_CHECK_EQ(output.landmark_size(), total_elements_);
+      cc->Outputs().Index(0).AddPacket(
+          MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmark>(input.landmark(ranges_[i].first))
+                  .At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          NormalizedLandmarkList output;
+          for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+            const NormalizedLandmark& input_landmark = input.landmark(j);
+            *output.add_landmark() = input_landmark;
+          }
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmarkList>(output).At(
+                  cc->InputTimestamp()));
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::vector<std::pair<int32, int32>> ranges_;
+  int32 max_range_end_ = -1;
+  int32 total_elements_ = 0;
+  bool element_only_ = false;
+  bool combine_outputs_ = false;
+};
+
+REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator);
+
+}  // namespace mediapipe
+
+// NOLINTNEXTLINE
+#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -0,0 +1,404 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+constexpr float kLocationVal = 3;
+
+class SplitNormalizedLandmarkListCalculatorTest : public ::testing::Test {
+ protected:
+  void TearDown() { expected_landmarks_.reset(); }
+
+  void PrepareNormalizedLandmarkList(int list_size) {
+    // Prepare input landmark list.
+    input_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    expected_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    for (int i = 0; i < list_size; ++i) {
+      NormalizedLandmark* landmark = input_landmarks_->add_landmark();
+      landmark->set_x(i * kLocationVal);
+      landmark->set_y(i * kLocationVal);
+      landmark->set_z(i * kLocationVal);
+      // Save the landmarks for comparison after the graph runs.
+      *expected_landmarks_->add_landmark() = *landmark;
+    }
+  }
+
+  void ValidateListOutput(std::vector<Packet>& output_packets,
+                          int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const NormalizedLandmark& expected_landmark =
+          expected_landmarks_->landmark(input_begin_index + i);
+      const NormalizedLandmark& result = output_landmarks.landmark(i);
+      EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+      EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+      EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+    }
+  }
+
+  void ValidateCombinedListOutput(std::vector<Packet>& output_packets,
+                                  int expected_elements,
+                                  std::vector<int>& input_begin_indices,
+                                  std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const NormalizedLandmark& expected_landmark =
+            expected_landmarks_->landmark(i);
+        const NormalizedLandmark& result =
+            output_landmarks.landmark(element_id);
+        EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+        EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+        EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+        element_id++;
+      }
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+
+    const NormalizedLandmark& output_landmark =
+        output_packets[0].Get<NormalizedLandmark>();
+    ASSERT_TRUE(output_landmark.IsInitialized());
+
+    const NormalizedLandmark& expected_landmark =
+        expected_landmarks_->landmark(input_begin_index);
+
+    EXPECT_FLOAT_EQ(expected_landmark.x(), output_landmark.x());
+    EXPECT_FLOAT_EQ(expected_landmark.y(), output_landmark.y());
+    EXPECT_FLOAT_EQ(expected_landmark.z(), output_landmark.z());
+  }
+
+  std::unique_ptr<NormalizedLandmarkList> input_landmarks_ = nullptr;
+  std::unique_ptr<NormalizedLandmarkList> expected_landmarks_ = nullptr;
+  std::unique_ptr<CalculatorRunner> runner_ = nullptr;
+};
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateListOutput(range_0_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/0);
+  ValidateListOutput(range_1_packets, /*expected_elements=*/3,
+                     /*input_begin_index=*/1);
+  ValidateListOutput(range_2_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 0 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because of an invalid range (begin == end).
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOutputStreamCountTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidCombineOutputsMultipleOutputsTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOverlappingRangesTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateElementOutput(range_0_packets,
+                        /*input_begin_index=*/0);
+  ValidateElementOutput(range_1_packets,
+                        /*input_begin_index=*/2);
+  ValidateElementOutput(range_2_packets,
+                        /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedListOutput(range_0_packets, /*expected_elements=*/3,
+                             input_begin_indices, input_end_indices);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       ElementOnlyDisablesVectorOutputs) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/color_convert_calculator.cc

@@ -38,9 +38,11 @@ void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
 
 constexpr char kRgbaInTag[] = "RGBA_IN";
 constexpr char kRgbInTag[] = "RGB_IN";
+constexpr char kBgraInTag[] = "BGRA_IN";
 constexpr char kGrayInTag[] = "GRAY_IN";
 constexpr char kRgbaOutTag[] = "RGBA_OUT";
 constexpr char kRgbOutTag[] = "RGB_OUT";
+constexpr char kBgraOutTag[] = "BGRA_OUT";
 constexpr char kGrayOutTag[] = "GRAY_OUT";
 }  // namespace
 
@@ -53,6 +55,8 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 //   GRAY -> RGB
 //   RGB  -> GRAY
 //   RGB  -> RGBA
+//   RGBA -> BGRA
+//   BGRA -> RGBA
 //
 // This calculator only supports a single input stream and output stream at a
 // time. If more than one input stream or output stream is present, the
@@ -63,11 +67,13 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 // Input streams:
 //   RGBA_IN:       The input video stream (ImageFrame, SRGBA).
 //   RGB_IN:        The input video stream (ImageFrame, SRGB).
+//   BGRA_IN:       The input video stream (ImageFrame, SBGRA).
 //   GRAY_IN:       The input video stream (ImageFrame, GRAY8).
 //
 // Output streams:
 //   RGBA_OUT:      The output video stream (ImageFrame, SRGBA).
 //   RGB_OUT:       The output video stream (ImageFrame, SRGB).
+//   BGRA_OUT:      The output video stream (ImageFrame, SBGRA).
 //   GRAY_OUT:      The output video stream (ImageFrame, GRAY8).
 class ColorConvertCalculator : public CalculatorBase {
  public:
@@ -113,6 +119,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
     cc->Inputs().Tag(kRgbInTag).Set<ImageFrame>();
   }
 
+  if (cc->Inputs().HasTag(kBgraInTag)) {
+    cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>();
+  }
+
   if (cc->Outputs().HasTag(kRgbOutTag)) {
     cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>();
   }
@@ -125,6 +135,10 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
     cc->Outputs().Tag(kRgbaOutTag).Set<ImageFrame>();
   }
 
+  if (cc->Outputs().HasTag(kBgraOutTag)) {
+    cc->Outputs().Tag(kBgraOutTag).Set<ImageFrame>();
+  }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -171,6 +185,16 @@ REGISTER_CALCULATOR(ColorConvertCalculator);
     return ConvertAndOutput(kRgbInTag, kRgbaOutTag, ImageFormat::SRGBA,
                             cv::COLOR_RGB2RGBA, cc);
   }
+  // BGRA -> RGBA
+  if (cc->Inputs().HasTag(kBgraInTag) && cc->Outputs().HasTag(kRgbaOutTag)) {
+    return ConvertAndOutput(kBgraInTag, kRgbaOutTag, ImageFormat::SRGBA,
+                            cv::COLOR_BGRA2RGBA, cc);
+  }
+  // RGBA -> BGRA
+  if (cc->Inputs().HasTag(kRgbaInTag) && cc->Outputs().HasTag(kBgraOutTag)) {
+    return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA,
+                            cv::COLOR_RGBA2BGRA, cc);
+  }
 
   return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
          << "Unsupported image format conversion.";

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -514,13 +514,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
     }
   }
 
-  return {
+  return {crop_width, crop_height, x_center, y_center, rotation};
-      .width = crop_width,
-      .height = crop_height,
-      .center_x = x_center,
-      .center_y = y_center,
-      .rotation = rotation,
-  };
 }
 
 ::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(

mediapipe/calculators/image/image_properties_calculator.cc

@@ -19,6 +19,11 @@
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
+namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+}  // namespace
+
 namespace mediapipe {
 
 // Extracts image properties from the input image and outputs the properties.
@@ -40,13 +45,14 @@ namespace mediapipe {
 class ImagePropertiesCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-    if (cc->Inputs().HasTag("IMAGE")) {
+              cc->Inputs().HasTag(kGpuBufferTag));
-      cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    if (cc->Inputs().HasTag(kImageFrameTag)) {
+      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU")) {
+    if (cc->Inputs().HasTag(kGpuBufferTag)) {
-      cc->Inputs().Tag("IMAGE_GPU").Set<::mediapipe::GpuBuffer>();
+      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
     }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
@@ -66,16 +72,17 @@ class ImagePropertiesCalculator : public CalculatorBase {
     int width;
     int height;
 
-    if (cc->Inputs().HasTag("IMAGE") && !cc->Inputs().Tag("IMAGE").IsEmpty()) {
+    if (cc->Inputs().HasTag(kImageFrameTag) &&
-      const auto& image = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
+      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
       width = image.Width();
       height = image.Height();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU") &&
+    if (cc->Inputs().HasTag(kGpuBufferTag) &&
-        !cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
       const auto& image =
-          cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
       width = image.width();
       height = image.height();
     }

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -47,6 +47,9 @@ namespace mediapipe {
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+
 int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
   switch (rotation) {
     case mediapipe::RotationMode_Mode_UNKNOWN:
@@ -95,7 +98,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Scales, rotates, and flips images horizontally or vertically.
 //
 // Input:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE: ImageFrame representing the input image.
 //   IMAGE_GPU: GpuBuffer representing the input image.
 //
@@ -113,7 +116,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   corresponding field in the calculator options.
 //
 // Output:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE - ImageFrame representing the output image.
 //   IMAGE_GPU - GpuBuffer representing the output image.
 //
@@ -152,7 +155,8 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Note: To enable horizontal or vertical flipping, specify them in the
 // calculator options. Flipping is applied after rotation.
 //
-// Note: Only scale mode STRETCH is currently supported on CPU.
+// Note: Input defines output, so only matchig types supported:
+// IMAGE -> IMAGE  or  IMAGE_GPU -> IMAGE_GPU
 //
 class ImageTransformationCalculator : public CalculatorBase {
  public:
@@ -186,7 +190,7 @@ class ImageTransformationCalculator : public CalculatorBase {
 
   bool use_gpu_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  GlCalculatorHelper helper_;
+  GlCalculatorHelper gpu_helper_;
   std::unique_ptr<QuadRenderer> rgb_renderer_;
   std::unique_ptr<QuadRenderer> yuv_renderer_;
   std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
@@ -197,21 +201,22 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 // static
 ::mediapipe::Status ImageTransformationCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  // Only one input can be set, and the output type must match.
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE"));
+    RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
+    RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
-    cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
-    cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+    cc->Outputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -259,7 +264,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   options_ = cc->Options<ImageTransformationCalculatorOptions>();
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
   }
 
@@ -300,7 +305,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     // Let the helper access the GL context information.
-    MP_RETURN_IF_ERROR(helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -328,18 +333,14 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+    if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
       return ::mediapipe::OkStatus();
     }
-    return helper_.RunInGlContext(
+    return gpu_helper_.RunInGlContext(
         [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
+    if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
       return ::mediapipe::OkStatus();
     }
     return RenderCpu(cc);
@@ -354,7 +355,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     QuadRenderer* rgb_renderer = rgb_renderer_.release();
     QuadRenderer* yuv_renderer = yuv_renderer_.release();
     QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release();
-    helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
+    gpu_helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
       if (rgb_renderer) {
         rgb_renderer->GlTeardown();
         delete rgb_renderer;
@@ -376,31 +377,41 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
 ::mediapipe::Status ImageTransformationCalculator::RenderCpu(
     CalculatorContext* cc) {
-  int input_width = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Width();
+  cv::Mat input_mat;
-  int input_height = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Height();
+  mediapipe::ImageFormat::Format format;
 
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  const auto& input = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-  cv::Mat input_mat = formats::MatView(&input_img);
+  input_mat = formats::MatView(&input);
-  cv::Mat scaled_mat;
+  format = input.Format();
 
+  const int input_width = input_mat.cols;
+  const int input_height = input_mat.rows;
   if (!output_height_ || !output_width_) {
     output_height_ = input_height;
     output_width_ = input_width;
   }
 
+  cv::Mat scaled_mat;
+  int output_width = output_width_;
+  int output_height = output_height_;
   if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
-    cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
+    int scale_flag =
+        input_mat.cols > output_width_ && input_mat.rows > output_height_
+            ? cv::INTER_AREA
+            : cv::INTER_LINEAR;
+    cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
+               0, 0, scale_flag);
   } else {
     const float scale =
         std::min(static_cast<float>(output_width_) / input_width,
                  static_cast<float>(output_height_) / input_height);
     const int target_width = std::round(input_width * scale);
     const int target_height = std::round(input_height * scale);
-
+    int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
     if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
       cv::Mat intermediate_mat;
       cv::resize(input_mat, intermediate_mat,
-                 cv::Size(target_width, target_height));
+                 cv::Size(target_width, target_height), 0, 0, scale_flag);
       const int top = (output_height_ - target_height) / 2;
       const int bottom = output_height_ - target_height - top;
       const int left = (output_width_ - target_width) / 2;
@@ -409,16 +420,13 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
                          options_.constant_padding() ? cv::BORDER_CONSTANT
                                                      : cv::BORDER_REPLICATE);
     } else {
-      cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height));
+      cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
-      output_width_ = target_width;
+                 0, 0, scale_flag);
-      output_height_ = target_height;
+      output_width = target_width;
+      output_height = target_height;
     }
   }
 
-  int output_width;
-  int output_height;
-  ComputeOutputDimensions(input_width, input_height, &output_width,
-                          &output_height);
   if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
     auto padding = absl::make_unique<std::array<float, 4>>();
     ComputeOutputLetterboxPadding(input_width, input_height, output_width,
@@ -444,10 +452,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   }
 
   std::unique_ptr<ImageFrame> output_frame(
-      new ImageFrame(input_img.Format(), output_width, output_height));
+      new ImageFrame(format, output_width, output_height));
   cv::Mat output_mat = formats::MatView(output_frame.get());
   flipped_mat.copyTo(output_mat);
-  cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_frame.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -455,8 +465,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  int input_width = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().width();
+  const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<GpuBuffer>();
-  int input_height = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().height();
+  const int input_width = input.width();
+  const int input_height = input.height();
 
   int output_width;
   int output_height;
@@ -472,7 +483,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         .Add(padding.release(), cc->InputTimestamp());
   }
 
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>();
   QuadRenderer* renderer = nullptr;
   GlTexture src1;
 
@@ -486,11 +496,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
                                  {"video_frame_y", "video_frame_uv"}));
     }
     renderer = yuv_renderer_.get();
-    src1 = helper_.CreateSourceTexture(input, 0);
+    src1 = gpu_helper_.CreateSourceTexture(input, 0);
   } else  // NOLINT(readability/braces)
 #endif    // iOS
   {
-    src1 = helper_.CreateSourceTexture(input);
+    src1 = gpu_helper_.CreateSourceTexture(input);
 #if defined(TEXTURE_EXTERNAL_OES)
     if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
       if (!ext_rgb_renderer_) {
@@ -516,10 +526,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   mediapipe::FrameRotation rotation =
       mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
 
-  auto dst = helper_.CreateDestinationTexture(output_width, output_height,
+  auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
-                                              input.format());
+                                                  input.format());
 
-  helper_.BindFramebuffer(dst);  // GL_TEXTURE0
+  gpu_helper_.BindFramebuffer(dst);  // GL_TEXTURE0
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 
@@ -534,8 +544,8 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   // Execute GL commands, before getting result.
   glFlush();
 
-  auto output = dst.GetFrame<GpuBuffer>();
+  auto output = dst.template GetFrame<GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 

mediapipe/calculators/image/recolor_calculator.cc

@@ -32,6 +32,11 @@
 
 namespace {
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kMaskCpuTag[] = "MASK";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -112,39 +117,41 @@ REGISTER_CALCULATOR(RecolorCalculator);
   bool use_gpu = false;
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("MASK_GPU")) {
+  if (cc->Inputs().HasTag(kMaskGpuTag)) {
-    cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("MASK")) {
+  if (cc->Inputs().HasTag(kMaskCpuTag)) {
-    cc->Inputs().Tag("MASK").Set<ImageFrame>();
+    cc->Inputs().Tag(kMaskCpuTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Outputs().HasTag("IMAGE_GPU")) {
+  if (cc->Outputs().HasTag(kGpuBufferTag)) {
-    cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Outputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Outputs().HasTag("IMAGE")) {
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
   // Confirm only one of the input streams is present.
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) ^
+            cc->Outputs().HasTag(kGpuBufferTag));
 
   if (use_gpu) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -158,7 +165,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 ::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -201,12 +208,12 @@ REGISTER_CALCULATOR(RecolorCalculator);
 }
 
 ::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
   // Get inputs and setup output.
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  const auto& input_img = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-  const auto& mask_img = cc->Inputs().Tag("MASK").Get<ImageFrame>();
+  const auto& mask_img = cc->Inputs().Tag(kMaskCpuTag).Get<ImageFrame>();
 
   cv::Mat input_mat = formats::MatView(&input_img);
   cv::Mat mask_mat = formats::MatView(&mask_img);
@@ -254,19 +261,21 @@ REGISTER_CALCULATOR(RecolorCalculator);
     }
   }
 
-  cc->Outputs().Tag("IMAGE").Add(output_img.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_img.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   // Get inputs and setup output.
-  const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
+  const Packet& input_packet = cc->Inputs().Tag(kGpuBufferTag).Value();
-  const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value();
+  const Packet& mask_packet = cc->Inputs().Tag(kMaskGpuTag).Value();
 
   const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
   const auto& mask_buffer = mask_packet.Get<mediapipe::GpuBuffer>();
@@ -296,7 +305,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 
   // Send result image in GPU packet.
   auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
   // Cleanup
   img_tex.Release();

mediapipe/calculators/tensorflow/BUILD

@@ -321,7 +321,7 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_lib_lite",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
     }),
     alwayslink = 1,
@@ -343,7 +343,7 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_lib_lite",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
     }),
     alwayslink = 1,
@@ -449,10 +449,10 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
         "//mediapipe:ios": [
-            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
         ],
     }),
     alwayslink = 1,
@@ -470,10 +470,10 @@ cc_library(
             "@org_tensorflow//tensorflow/core:core",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
         "//mediapipe:ios": [
-            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
         ],
     }),
 )
@@ -496,11 +496,11 @@ cc_library(
             "@org_tensorflow//tensorflow/core:core",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
             "//mediapipe/android/file/base",
         ],
         "//mediapipe:ios": [
-            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
             "//mediapipe/android/file/base",
         ],
     }),
@@ -525,11 +525,11 @@ cc_library(
             "@org_tensorflow//tensorflow/core:core",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
             "//mediapipe/android/file/base",
         ],
         "//mediapipe:ios": [
-            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
             "//mediapipe/android/file/base",
         ],
     }),
@@ -637,7 +637,7 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_lib_lite",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
     }),
     alwayslink = 1,
@@ -673,7 +673,7 @@ cc_library(
             "@org_tensorflow//tensorflow/core:framework",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_lib_lite",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
         ],
     }),
     alwayslink = 1,
@@ -1109,11 +1109,11 @@ cc_test(
             "@org_tensorflow//tensorflow/core:direct_session",
         ],
         "//mediapipe:android": [
-            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_with_ops_lite_proto_no_rtti_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib",
-            "@org_tensorflow//tensorflow/core:android_tensorflow_test_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib",
         ],
         "//mediapipe:ios": [
-            "@org_tensorflow//tensorflow/core:ios_tensorflow_test_lib",
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib",
         ],
     }),
 )

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -244,7 +244,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
   ::mediapipe::Status VerifySequence() {
     std::string error_msg = "Missing features - ";
     bool all_present = true;
-    for (auto iter : features_present_) {
+    for (const auto& iter : features_present_) {
       if (!iter.second) {
         all_present = false;
         absl::StrAppend(&error_msg, iter.first, ", ");

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -126,7 +126,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -121,7 +121,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tflite/BUILD

@@ -198,6 +198,7 @@ cc_test(
 cc_library(
     name = "util",
     hdrs = ["util.h"],
+    visibility = ["//visibility:public"],
     alwayslink = 1,
 )
 
@@ -243,6 +244,7 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
         ],
         "//conditions:default": [
+            "//mediapipe/util/tflite:tflite_gpu_runner",
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gpu_buffer",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
@@ -524,16 +526,16 @@ cc_test(
         ":tflite_converter_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
-        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
-        "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:validate_type",
         "@com_google_absl//absl/memory",
         "@org_tensorflow//tensorflow/lite:framework",
-        "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
     ],
 )
 

mediapipe/calculators/tflite/ssd_anchors_calculator.cc

@@ -26,8 +26,12 @@ namespace {
 
 float CalculateScale(float min_scale, float max_scale, int stride_index,
                      int num_strides) {
-  return min_scale +
+  if (num_strides == 1) {
-         (max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0f);
+    return (min_scale + max_scale) * 0.5f;
+  } else {
+    return min_scale +
+           (max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0f);
+  }
 }
 
 }  // namespace
@@ -114,7 +118,7 @@ REGISTER_CALCULATOR(SsdAnchorsCalculator);
   }
 
   int layer_id = 0;
-  while (layer_id < options.strides_size()) {
+  while (layer_id < options.num_layers()) {
     std::vector<float> anchor_height;
     std::vector<float> anchor_width;
     std::vector<float> aspect_ratios;

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -63,10 +63,16 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
 typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
     ColMajorMatrixXf;
 
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
+constexpr char kMatrixTag[] = "MATRIX";
 }  // namespace
 
 namespace mediapipe {
 
+namespace {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlProgram;
@@ -85,6 +91,8 @@ struct GPUData {
 };
 #endif
 
+}  // namespace
+
 // Calculator for normalizing and converting an ImageFrame or Matrix
 // into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
 // or MTLBuffer.
@@ -124,6 +132,9 @@ struct GPUData {
 //  GPU tensors are currently only supported on mobile platforms.
 //  This calculator uses FixedSizeInputStreamHandler by default.
 //
+// Note: Input defines output, so only these type sets are supported:
+// IMAGE -> TENSORS | IMAGE_GPU -> TENSORS_GPU | MATRIX -> TENSORS
+//
 class TfLiteConverterCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
@@ -138,9 +149,9 @@ class TfLiteConverterCalculator : public CalculatorBase {
   template <class T>
   ::mediapipe::Status NormalizeImage(const ImageFrame& image_frame,
                                      bool zero_center, bool flip_vertically,
-                                     float* tensor_buffer);
+                                     float* tensor_ptr);
   ::mediapipe::Status CopyMatrixToTensor(const Matrix& matrix,
-                                         float* tensor_buffer);
+                                         float* tensor_ptr);
   ::mediapipe::Status ProcessCPU(CalculatorContext* cc);
   ::mediapipe::Status ProcessGPU(CalculatorContext* cc);
 
@@ -157,6 +168,9 @@ class TfLiteConverterCalculator : public CalculatorBase {
   bool initialized_ = false;
   bool use_gpu_ = false;
   bool zero_center_ = true;  // normalize range to [-1,1] | otherwise [0,1]
+  bool use_custom_normalization_ = false;
+  float custom_div_ = -1.0f;
+  float custom_sub_ = -1.0f;
   bool flip_vertically_ = false;
   bool row_major_matrix_ = false;
   bool use_quantized_tensors_ = false;
@@ -166,33 +180,36 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::GetContract(
     CalculatorContract* cc) {
-  const bool has_image_tag = cc->Inputs().HasTag("IMAGE");
-  const bool has_image_gpu_tag = cc->Inputs().HasTag("IMAGE_GPU");
-  const bool has_matrix_tag = cc->Inputs().HasTag("MATRIX");
   // Confirm only one of the input streams is present.
-  RET_CHECK(has_image_tag ^ has_image_gpu_tag ^ has_matrix_tag &&
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-            !(has_image_tag && has_image_gpu_tag && has_matrix_tag));
+            cc->Inputs().HasTag(kGpuBufferTag) ^
+            cc->Inputs().HasTag(kMatrixTag));
 
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-  if (cc->Inputs().HasTag("MATRIX")) cc->Inputs().Tag("MATRIX").Set<Matrix>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+  }
+  if (cc->Inputs().HasTag(kMatrixTag)) {
+    cc->Inputs().Tag(kMatrixTag).Set<Matrix>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
+  if (cc->Outputs().HasTag(kTensorsTag)) {
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -216,8 +233,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   MP_RETURN_IF_ERROR(LoadOptions(cc));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU") ||
+  if (cc->Inputs().HasTag(kGpuBufferTag) ||
-      cc->Outputs().HasTag("IMAGE_OUT_GPU")) {
+      cc->Outputs().HasTag(kGpuBufferTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     use_gpu_ = true;
 #else
@@ -227,8 +244,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   if (use_gpu_) {
     // Cannot mix CPU/GPU streams.
-    RET_CHECK(cc->Inputs().HasTag("IMAGE_GPU") &&
+    RET_CHECK(cc->Inputs().HasTag(kGpuBufferTag) &&
-              cc->Outputs().HasTag("TENSORS_GPU"));
+              cc->Outputs().HasTag(kTensorsGpuTag));
     // Cannot use quantization.
     use_quantized_tensors_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -248,7 +265,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-    // GpuBuffer to tflite::gpu::GlBuffer conversion.
+    if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     if (!initialized_) {
       MP_RETURN_IF_ERROR(InitGpu(cc));
       initialized_ = true;
@@ -259,7 +278,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     // Convert to CPU tensors or Matrix type.
     MP_RETURN_IF_ERROR(ProcessCPU(cc));
   }
-
   return ::mediapipe::OkStatus();
 }
 
@@ -275,24 +293,29 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::ProcessCPU(
     CalculatorContext* cc) {
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
+    if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     // CPU ImageFrame to TfLiteTensor conversion.
 
-    const auto& image_frame = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+    const auto& image_frame =
+        cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
     const int height = image_frame.Height();
     const int width = image_frame.Width();
     const int channels = image_frame.NumberOfChannels();
     const int channels_preserved = std::min(channels, max_num_channels_);
+    const mediapipe::ImageFormat::Format format = image_frame.Format();
 
     if (!initialized_) {
-      if (!(image_frame.Format() == mediapipe::ImageFormat::SRGBA ||
+      if (!(format == mediapipe::ImageFormat::SRGBA ||
-            image_frame.Format() == mediapipe::ImageFormat::SRGB ||
+            format == mediapipe::ImageFormat::SRGB ||
-            image_frame.Format() == mediapipe::ImageFormat::GRAY8 ||
+            format == mediapipe::ImageFormat::GRAY8 ||
-            image_frame.Format() == mediapipe::ImageFormat::VEC32F1))
+            format == mediapipe::ImageFormat::VEC32F1))
         RET_CHECK_FAIL() << "Unsupported CPU input format.";
       TfLiteQuantization quant;
       if (use_quantized_tensors_) {
-        RET_CHECK(image_frame.Format() != mediapipe::ImageFormat::VEC32F1)
+        RET_CHECK(format != mediapipe::ImageFormat::VEC32F1)
             << "Only 8-bit input images are supported for quantization.";
         quant.type = kTfLiteAffineQuantization;
         quant.params = nullptr;
@@ -349,12 +372,15 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
-  } else if (cc->Inputs().HasTag("MATRIX")) {
+        .Add(output_tensors.release(), cc->InputTimestamp());
+  } else if (cc->Inputs().HasTag(kMatrixTag)) {
+    if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     // CPU Matrix to TfLiteTensor conversion.
-
+    const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
-    const auto& matrix = cc->Inputs().Tag("MATRIX").Get<Matrix>();
     const int height = matrix.rows();
     const int width = matrix.cols();
     const int channels = 1;
@@ -371,15 +397,16 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     interpreter_->ResizeInputTensor(tensor_idx, {height, width, channels});
     interpreter_->AllocateTensors();
 
-    float* tensor_buffer = tensor->data.f;
+    float* tensor_ptr = tensor->data.f;
-    RET_CHECK(tensor_buffer);
+    RET_CHECK(tensor_ptr);
 
-    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_buffer));
+    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_ptr));
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -389,7 +416,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // GpuBuffer to tflite::gpu::GlBuffer conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   MP_RETURN_IF_ERROR(
       gpu_helper_.RunInGlContext([this, &input]() -> ::mediapipe::Status {
         // Convert GL texture into TfLite GlBuffer (SSBO).
@@ -421,11 +449,12 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
         return ::mediapipe::OkStatus();
       }));
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
 
   id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
@@ -457,7 +486,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
                     commandBuffer:command_buffer];
 
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #else
   RET_CHECK_FAIL() << "GPU processing is not enabled.";
@@ -469,7 +498,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 ::mediapipe::Status TfLiteConverterCalculator::InitGpu(CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
   // Get input image sizes.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   mediapipe::ImageFormat::Format format =
       mediapipe::ImageFormatForGpuBufferFormat(input.format());
   gpu_data_out_ = absl::make_unique<GPUData>();
@@ -600,6 +630,11 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   // Get data normalization mode.
   zero_center_ = options.zero_center();
 
+  // Custom div and sub values.
+  use_custom_normalization_ = options.use_custom_normalization();
+  custom_div_ = options.custom_div();
+  custom_sub_ = options.custom_sub();
+
   // Get y-flip mode.
   flip_vertically_ = options.flip_vertically();
 
@@ -612,7 +647,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   CHECK_LE(max_num_channels_, 4);
   CHECK_NE(max_num_channels_, 2);
 #if defined(MEDIAPIPE_IOS)
-  if (cc->Inputs().HasTag("IMAGE_GPU"))
+  if (cc->Inputs().HasTag(kGpuBufferTag))
     // Currently on iOS, tflite gpu input tensor must be 4 channels,
     // so input image must be 4 channels also (checked in InitGpu).
     max_num_channels_ = 4;
@@ -627,7 +662,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 template <class T>
 ::mediapipe::Status TfLiteConverterCalculator::NormalizeImage(
     const ImageFrame& image_frame, bool zero_center, bool flip_vertically,
-    float* tensor_buffer) {
+    float* tensor_ptr) {
   const int height = image_frame.Height();
   const int width = image_frame.Width();
   const int channels = image_frame.NumberOfChannels();
@@ -635,7 +670,13 @@ template <class T>
   const int channels_ignored = channels - channels_preserved;
 
   float div, sub;
-  if (zero_center) {
+
+  if (use_custom_normalization_) {
+    RET_CHECK_GT(custom_div_, 0.0f);
+    RET_CHECK_GE(custom_sub_, 0.0f);
+    div = custom_div_;
+    sub = custom_sub_;
+  } else if (zero_center) {
     // [-1,1]
     div = 127.5f;
     sub = 1.0f;
@@ -651,7 +692,7 @@ template <class T>
         (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
     for (int j = 0; j < width; ++j) {
       for (int c = 0; c < channels_preserved; ++c) {
-        *tensor_buffer++ = *image_ptr++ / div - sub;
+        *tensor_ptr++ = *image_ptr++ / div - sub;
       }
       image_ptr += channels_ignored;
     }
@@ -661,14 +702,14 @@ template <class T>
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::CopyMatrixToTensor(
-    const Matrix& matrix, float* tensor_buffer) {
+    const Matrix& matrix, float* tensor_ptr) {
   if (row_major_matrix_) {
-    auto matrix_map = Eigen::Map<RowMajorMatrixXf>(tensor_buffer, matrix.rows(),
+    auto matrix_map =
-                                                   matrix.cols());
+        Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   } else {
-    auto matrix_map = Eigen::Map<ColMajorMatrixXf>(tensor_buffer, matrix.rows(),
+    auto matrix_map =
-                                                   matrix.cols());
+        Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   }
 

mediapipe/calculators/tflite/tflite_converter_calculator.proto

@@ -28,6 +28,16 @@ message TfLiteConverterCalculatorOptions {
   // Ignored if using quantization.
   optional bool zero_center = 1 [default = true];
 
+  // Custom settings to override the internal scaling factors `div` and `sub`.
+  // Both values must be set to non-negative values. Will only take effect on
+  // CPU AND when |use_custom_normalization| is set to true. When these custom
+  // values take effect, the |zero_center| setting above will be overriden, and
+  // the normalized_value will be calculated as:
+  // normalized_value = input / custom_div - custom_sub.
+  optional bool use_custom_normalization = 6 [default = false];
+  optional float custom_div = 7 [default = -1.0];
+  optional float custom_sub = 8 [default = -1.0];
+
   // Whether the input image should be flipped vertically (along the
   // y-direction). This is useful, for example, when the input image is defined
   // with a coordinate system where the origin is at the bottom-left corner

mediapipe/calculators/tflite/tflite_converter_calculator_test.cc

@@ -19,6 +19,9 @@
 #include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -28,7 +31,6 @@
 #include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
-
 namespace {
 
 constexpr char kTransposeOptionsString[] =
@@ -196,4 +198,55 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
   }
 }
 
+TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) {
+  CalculatorGraph graph;
+  // Run the calculator and verify that one output is generated.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "input_image"
+        node {
+          calculator: "TfLiteConverterCalculator"
+          input_stream: "IMAGE:input_image"
+          output_stream: "TENSORS:tensor"
+          options {
+            [mediapipe.TfLiteConverterCalculatorOptions.ext] {
+              row_major_matrix: true
+              use_custom_normalization: true
+              custom_div: 2.0
+              custom_sub: 33.0
+            }
+          }
+        }
+      )");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("tensor", &graph_config, &output_packets);
+
+  // Run the graph.
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
+  cv::Mat mat = ::mediapipe::formats::MatView(input_image.get());
+  mat.at<uint8>(0, 0) = 200;
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_image", Adopt(input_image.release()).At(Timestamp(0))));
+
+  // Wait until the calculator done processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(1, output_packets.size());
+
+  // Get and process results.
+  const std::vector<TfLiteTensor>& tensor_vec =
+      output_packets[0].Get<std::vector<TfLiteTensor>>();
+  EXPECT_EQ(1, tensor_vec.size());
+
+  const TfLiteTensor* tensor = &tensor_vec[0];
+  EXPECT_EQ(kTfLiteFloat32, tensor->type);
+  EXPECT_FLOAT_EQ(67.0f, *tensor->data.f);
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  MP_ASSERT_OK(graph.CloseInputStream("input_image"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
 }  // namespace mediapipe

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -23,9 +23,9 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
 #include "mediapipe/util/cpu_util.h"
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
@@ -36,6 +36,7 @@
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/util/tflite/tflite_gpu_runner.h"
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
@@ -56,7 +57,10 @@
 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
 #include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
 #endif  // iOS
+
+#if !defined(MEDIAPIPE_EDGE_TPU)
 #include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h"
+#endif  // !EDGETPU
 #if defined(MEDIAPIPE_ANDROID)
 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
 #endif  // ANDROID
@@ -75,6 +79,9 @@ typedef id<MTLBuffer> GpuTensor;
 
 // Round up n to next multiple of m.
 size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 #if defined(MEDIAPIPE_EDGE_TPU)
@@ -112,16 +119,18 @@ using ::tflite::gpu::gl::GlBuffer;
 #endif
 
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
+namespace {
 struct GPUData {
   int elements = 1;
   GpuTensor buffer;
   ::tflite::gpu::BHWC shape;
 };
+}  // namespace
 #endif
 
 // Returns number of threads to configure XNNPACK delegate with.
 // (Equal to user provided value if specified.  Otherwise, it returns number of
-// high cores (hard-coded to 1 for __EMSCRIPTEN__))
+// high cores (hard-coded to 1 for Emscripten without Threads extension))
 int GetXnnpackNumThreads(
     const mediapipe::TfLiteInferenceCalculatorOptions& opts) {
   static constexpr int kDefaultNumThreads = -1;
@@ -129,11 +138,11 @@ int GetXnnpackNumThreads(
       opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
     return opts.delegate().xnnpack().num_threads();
   }
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
   return InferHigherCoreIds().size();
 #else
   return 1;
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 }
 
 // Calculator Header Section
@@ -219,6 +228,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   ::mediapipe::Status LoadModel(CalculatorContext* cc);
   ::mediapipe::StatusOr<Packet> GetModelAsPacket(const CalculatorContext& cc);
   ::mediapipe::Status LoadDelegate(CalculatorContext* cc);
+  ::mediapipe::Status InitTFLiteGPURunner();
 
   Packet model_packet_;
   std::unique_ptr<tflite::Interpreter> interpreter_;
@@ -228,6 +238,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
 #elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
@@ -245,6 +256,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   bool gpu_input_ = false;
   bool gpu_output_ = false;
   bool use_quantized_tensors_ = false;
+
+  bool use_advanced_gpu_api_ = false;
 };
 REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
@@ -252,10 +265,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Inputs().HasTag(kTensorsTag) ^
-            cc->Inputs().HasTag("TENSORS_GPU"));
+            cc->Inputs().HasTag(kTensorsGpuTag));
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   const auto& options =
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
@@ -266,26 +279,26 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   bool use_gpu =
       options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS"))
+  if (cc->Inputs().HasTag(kTensorsTag))
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU input is compatible with GPU delegate only.";
 
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
+  if (cc->Outputs().HasTag(kTensorsTag))
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU output is compatible with GPU delegate only.";
 
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -320,27 +333,31 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
   gpu_inference_ = options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_input_ = true;
     gpu_inference_ = true;  // Inference must be on GPU also.
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_output_ = true;
-    RET_CHECK(cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU output must also have GPU Input.";
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
+  const auto& calculator_opts =
+      cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
+  use_advanced_gpu_api_ = false;
+
   MP_RETURN_IF_ERROR(LoadModel(cc));
 
   if (gpu_inference_) {
@@ -352,8 +369,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+    MP_RETURN_IF_ERROR(
-        [this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
+        gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
+          return use_advanced_gpu_api_ ? InitTFLiteGPURunner()
+                                       : LoadDelegate(cc);
+        }));
+    if (use_advanced_gpu_api_) return ::mediapipe::OkStatus();
 #else
     MP_RETURN_IF_ERROR(LoadDelegate(cc));
 #endif
@@ -365,13 +386,57 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TfLiteInferenceCalculator::InitTFLiteGPURunner() {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  // Create and bind OpenGL buffers for outputs.
+  // These buffers are created onve and later their ids are jut passed to the
+  // calculator outputs.
+
+  gpu_data_out_.resize(tflite_gpu_runner_->outputs_size());
+  for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) {
+    gpu_data_out_[i] = absl::make_unique<GPUData>();
+    ASSIGN_OR_RETURN(gpu_data_out_[i]->elements,
+                     tflite_gpu_runner_->GetOutputElements(i));
+    // Create and bind input buffer.
+    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+        gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
+  }
+  RET_CHECK_CALL(tflite_gpu_runner_->Build());
+#endif
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TfLiteInferenceCalculator::Process(CalculatorContext* cc) {
   // 1. Receive pre-processed tensor inputs.
-  if (gpu_input_) {
+  if (use_advanced_gpu_api_) {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+    const auto& input_tensors =
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
+    RET_CHECK(!input_tensors.empty());
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+        [this, &input_tensors]() -> ::mediapipe::Status {
+          for (int i = 0; i < input_tensors.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
+                input_tensors[i].id(), i));
+          }
+          for (int i = 0; i < gpu_data_out_.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
+                gpu_data_out_[i]->buffer.id(), i));
+          }
+          return ::mediapipe::OkStatus();
+        }));
+#endif
+  } else if (gpu_input_) {
     // Read GPU input into SSBO.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
@@ -385,8 +450,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
+    if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     // Explicit copy input with conversion float 32 bits to 16 bits.
     gpu_data_in_.resize(input_tensors.size());
@@ -411,9 +479,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif
   } else {
+    if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
     // Read CPU input into tensors.
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+        cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     for (int i = 0; i < input_tensors.size(); ++i) {
       const TfLiteTensor* input_tensor = &input_tensors[i];
@@ -437,7 +508,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
-          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          if (use_advanced_gpu_api_) {
+            RET_CHECK(tflite_gpu_runner_->Invoke().ok());
+          } else {
+            RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          }
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
@@ -448,7 +523,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   }
 
   // 3. Output processed tensors.
-  if (gpu_output_) {
+  if (use_advanced_gpu_api_) {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
+    output_tensors->resize(gpu_data_out_.size());
+    for (int i = 0; i < gpu_data_out_.size(); ++i) {
+      output_tensors->at(i) = gpu_data_out_[i]->buffer.MakeRef();
+    }
+    cc->Outputs()
+        .Tag(kTensorsGpuTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
+#endif
+  } else if (gpu_output_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
@@ -464,7 +550,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           return ::mediapipe::OkStatus();
         }));
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
     // Output result tensors (GPU).
@@ -488,7 +574,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     [convert_command endEncoding];
     [command_buffer commit];
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
@@ -501,8 +587,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
       output_tensors->emplace_back(*tensor);
     }
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -513,6 +600,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     if (gpu_inference_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
+        interpreter_ = nullptr;
         delegate_ = nullptr;
         for (int i = 0; i < gpu_data_in_.size(); ++i) {
           gpu_data_in_[i].reset();
@@ -523,6 +611,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         return ::mediapipe::OkStatus();
       }));
 #elif defined(MEDIAPIPE_IOS)
+      interpreter_ = nullptr;
       delegate_ = nullptr;
       for (int i = 0; i < gpu_data_in_.size(); ++i) {
         gpu_data_in_[i].reset();
@@ -532,6 +621,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       }
 #endif
     } else {
+      interpreter_ = nullptr;
       delegate_ = nullptr;
     }
   }
@@ -554,6 +644,20 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
                       .Tag("CUSTOM_OP_RESOLVER")
                       .Get<tflite::ops::builtin::BuiltinOpResolver>();
   }
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  if (use_advanced_gpu_api_) {
+    tflite::gpu::InferenceOptions options;
+    options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
+    options.priority2 = tflite::gpu::InferencePriority::AUTO;
+    options.priority3 = tflite::gpu::InferencePriority::AUTO;
+    options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+    tflite_gpu_runner_ =
+        std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
+    return tflite_gpu_runner_->InitializeWithModel(model, op_resolver);
+  }
+#endif
+
 #if defined(MEDIAPIPE_EDGE_TPU)
   interpreter_ =
       BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
@@ -643,6 +747,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
                                    calculator_opts.delegate().has_xnnpack();
 #endif  // __EMSCRIPTEN__
 
+#if !defined(MEDIAPIPE_EDGE_TPU)
     if (xnnpack_requested) {
       TfLiteXNNPackDelegateOptions xnnpack_opts{};
       xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts);
@@ -651,6 +756,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
                    kTfLiteOk);
     }
+#endif  // !EDGETPU
 
     // Return, no need for GPU delegate below.
     return ::mediapipe::OkStatus();

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -42,7 +42,11 @@ message TfLiteInferenceCalculatorOptions {
     message TfLite {}
     // Delegate to run GPU inference depending on the device.
     // (Can use OpenGl, OpenCl, Metal depending on the device.)
-    message Gpu {}
+    message Gpu {
+      // Experimental, Android/Linux only. Use TFLite GPU delegate API2 for
+      // the NN inference.
+      optional bool use_advanced_gpu_api = 1 [default = false];
+    }
     // Android only.
     message Nnapi {}
     message Xnnpack {

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -47,10 +47,11 @@
 #endif  // iOS
 
 namespace {
-
 constexpr int kNumInputTensorsWithAnchors = 3;
 constexpr int kNumCoordsPerBox = 4;
 
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -99,7 +100,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
                                int num_boxes, float* raw_anchors) {
   CHECK_EQ(anchors.size(), num_boxes);
   int box = 0;
-  for (auto anchor : anchors) {
+  for (const auto& anchor : anchors) {
     raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center();
     raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center();
     raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h();
@@ -200,13 +201,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("TENSORS")) {
+  if (cc->Inputs().HasTag(kTensorsTag)) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -236,7 +237,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     gpu_input_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -258,8 +259,8 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::Process(
     CalculatorContext* cc) {
-  if ((!gpu_input_ && cc->Inputs().Tag("TENSORS").IsEmpty()) ||
+  if ((!gpu_input_ && cc->Inputs().Tag(kTensorsTag).IsEmpty()) ||
-      (gpu_input_ && cc->Inputs().Tag("TENSORS_GPU").IsEmpty())) {
+      (gpu_input_ && cc->Inputs().Tag(kTensorsGpuTag).IsEmpty())) {
     return ::mediapipe::OkStatus();
   }
 
@@ -284,7 +285,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU(
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
 
   if (input_tensors.size() == 2 ||
       input_tensors.size() == kNumInputTensorsWithAnchors) {
@@ -402,7 +403,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc,
@@ -466,7 +467,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 #elif defined(MEDIAPIPE_IOS)
 
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   // Copy inputs.

mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc

@@ -49,6 +49,16 @@ int NumGroups(const int size, const int group_size) {  // NOLINT
 float Clamp(float val, float min, float max) {
   return std::min(std::max(val, min), max);
 }
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
+constexpr char kSizeImageTag[] = "REFERENCE_IMAGE";
+constexpr char kSizeImageGpuTag[] = "REFERENCE_IMAGE_GPU";
+constexpr char kMaskTag[] = "MASK";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
+constexpr char kPrevMaskTag[] = "PREV_MASK";
+constexpr char kPrevMaskGpuTag[] = "PREV_MASK_GPU";
+
 }  // namespace
 
 namespace mediapipe {
@@ -67,7 +77,10 @@ using ::tflite::gpu::gl::GlShader;
 // Performs optional upscale to REFERENCE_IMAGE dimensions if provided,
 // otherwise the mask is the same size as input tensor.
 //
-// Produces result as an RGBA image, with the mask in both R & A channels.
+// Produces result as an RGBA image, with the mask in both R & A channels. The
+// value of each pixel is the probability of the specified class after softmax,
+// scaled to 255 on CPU. The class can be specified through the
+// |output_layer_index| option.
 //
 // Inputs:
 //   One of the following TENSORS tags:
@@ -148,39 +161,39 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   bool use_gpu = false;
 
   // Inputs CPU.
-  if (cc->Inputs().HasTag("TENSORS")) {
+  if (cc->Inputs().HasTag(kTensorsTag)) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
-  if (cc->Inputs().HasTag("PREV_MASK")) {
+  if (cc->Inputs().HasTag(kPrevMaskTag)) {
-    cc->Inputs().Tag("PREV_MASK").Set<ImageFrame>();
+    cc->Inputs().Tag(kPrevMaskTag).Set<ImageFrame>();
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
-    cc->Inputs().Tag("REFERENCE_IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kSizeImageTag).Set<ImageFrame>();
   }
 
   // Inputs GPU.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GlBuffer>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GlBuffer>>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("PREV_MASK_GPU")) {
+  if (cc->Inputs().HasTag(kPrevMaskGpuTag)) {
-    cc->Inputs().Tag("PREV_MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kPrevMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
-    cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kSizeImageGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
   // Outputs.
-  if (cc->Outputs().HasTag("MASK")) {
+  if (cc->Outputs().HasTag(kMaskTag)) {
-    cc->Outputs().Tag("MASK").Set<ImageFrame>();
+    cc->Outputs().Tag(kMaskTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Outputs().HasTag("MASK_GPU")) {
+  if (cc->Outputs().HasTag(kMaskGpuTag)) {
-    cc->Outputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Outputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -197,7 +210,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -255,23 +268,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessCpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK") &&
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskTag) &&
-                             !cc->Inputs().Tag("PREV_MASK").IsEmpty();
+                             !cc->Inputs().Tag(kPrevMaskTag).IsEmpty();
   const ImageFrame placeholder;
-  const auto& input_mask = has_prev_mask
+  const auto& input_mask =
-                               ? cc->Inputs().Tag("PREV_MASK").Get<ImageFrame>()
+      has_prev_mask ? cc->Inputs().Tag(kPrevMaskTag).Get<ImageFrame>()
-                               : placeholder;
+                    : placeholder;
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
-    const auto& input_image =
+    const auto& input_image = cc->Inputs().Tag(kSizeImageTag).Get<ImageFrame>();
-        cc->Inputs().Tag("REFERENCE_IMAGE").Get<ImageFrame>();
     output_width = input_image.Width();
     output_height = input_image.Height();
   }
@@ -353,7 +365,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
       ImageFormat::SRGBA, output_width, output_height);
   cv::Mat output_mat = formats::MatView(output_mask.get());
   large_mask_mat.copyTo(output_mat);
-  cc->Outputs().Tag("MASK").Add(output_mask.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -364,23 +376,23 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 // 3. upsample small mask into output mask to be same size as input image
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessGpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GlBuffer>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GlBuffer>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK_GPU") &&
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskGpuTag) &&
-                             !cc->Inputs().Tag("PREV_MASK_GPU").IsEmpty();
+                             !cc->Inputs().Tag(kPrevMaskGpuTag).IsEmpty();
   const auto& input_mask =
       has_prev_mask
-          ? cc->Inputs().Tag("PREV_MASK_GPU").Get<mediapipe::GpuBuffer>()
+          ? cc->Inputs().Tag(kPrevMaskGpuTag).Get<mediapipe::GpuBuffer>()
           : mediapipe::GpuBuffer();
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
     const auto& input_image =
-        cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+        cc->Inputs().Tag(kSizeImageGpuTag).Get<mediapipe::GpuBuffer>();
     output_width = input_image.width();
     output_height = input_image.height();
   }
@@ -441,7 +453,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   // Send out image as GPU packet.
   auto output_image = output_texture.GetFrame<mediapipe::GpuBuffer>();
   cc->Outputs()
-      .Tag("MASK_GPU")
+      .Tag(kMaskGpuTag)
       .Add(output_image.release(), cc->InputTimestamp());
 
   // Cleanup

mediapipe/calculators/util/BUILD

@@ -276,6 +276,41 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "clock_timestamp_calculator",
+    srcs = ["clock_timestamp_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/deps:clock",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/time",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "clock_latency_calculator",
+    srcs = ["clock_latency_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/time",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "annotation_overlay_calculator",
     srcs = ["annotation_overlay_calculator.cc"],

mediapipe/calculators/util/clock_latency_calculator.cc

@@ -0,0 +1,116 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/time/time.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace {
+// Tag name for reference signal.
+constexpr char kReferenceTag[] = "REFERENCE";
+}  // namespace
+
+// A calculator that diffs multiple input absl::Time streams against a
+// reference Time stream, and outputs the resulting absl::Duration's. Useful
+// in combination with ClockTimestampCalculator to be able to determine the
+// latency between two different points in a graph.
+//
+// Inputs:  At least one non-reference Time stream is required.
+//   0- Time stream 0
+//   1- Time stream 1
+//   ...
+//   N- Time stream N
+//   REFERENCE_SIGNAL (required): The Time stream by which all others are
+//     compared. Should be the stream from which our other streams were
+//     computed, in order to provide meaningful latency results.
+//
+// Outputs:
+//   0- Duration from REFERENCE_SIGNAL to input stream 0
+//   1- Duration from REFERENCE_SIGNAL to input stream 1
+//   ...
+//   N- Duration from REFERENCE_SIGNAL to input stream N
+//
+// Example config:
+// node {
+//   calculator: "ClockLatencyCalculator"
+//   input_stream: "packet_clocktime_stream_0"
+//   input_stream: "packet_clocktime_stream_1"
+//   input_stream: "packet_clocktime_stream_2"
+//   input_stream: "REFERENCE_SIGNAL: packet_clocktime_stream_reference"
+//   output_stream: "packet_latency_stream_0"
+//   output_stream: "packet_latency_stream_1"
+//   output_stream: "packet_latency_stream_2"
+// }
+//
+class ClockLatencyCalculator : public CalculatorBase {
+ public:
+  ClockLatencyCalculator() {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  int64 num_packet_streams_ = -1;
+};
+REGISTER_CALCULATOR(ClockLatencyCalculator);
+
+::mediapipe::Status ClockLatencyCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK_GT(cc->Inputs().NumEntries(), 1);
+
+  int64 num_packet_streams = cc->Inputs().NumEntries() - 1;
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), num_packet_streams);
+
+  for (int64 i = 0; i < num_packet_streams; ++i) {
+    cc->Inputs().Index(i).Set<absl::Time>();
+    cc->Outputs().Index(i).Set<absl::Duration>();
+  }
+  cc->Inputs().Tag(kReferenceTag).Set<absl::Time>();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ClockLatencyCalculator::Open(CalculatorContext* cc) {
+  // Direct passthrough, as far as timestamp and bounds are concerned.
+  cc->SetOffset(TimestampDiff(0));
+  num_packet_streams_ = cc->Inputs().NumEntries() - 1;
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ClockLatencyCalculator::Process(CalculatorContext* cc) {
+  // Get reference time.
+  RET_CHECK(!cc->Inputs().Tag(kReferenceTag).IsEmpty());
+  const absl::Time& reference_time =
+      cc->Inputs().Tag(kReferenceTag).Get<absl::Time>();
+
+  // Push Duration packets for every input stream we have.
+  for (int64 i = 0; i < num_packet_streams_; ++i) {
+    if (!cc->Inputs().Index(i).IsEmpty()) {
+      const absl::Time& input_stream_time =
+          cc->Inputs().Index(i).Get<absl::Time>();
+      cc->Outputs().Index(i).AddPacket(
+          MakePacket<absl::Duration>(input_stream_time - reference_time)
+              .At(cc->InputTimestamp()));
+    }
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/clock_timestamp_calculator.cc

@@ -0,0 +1,108 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/time/time.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/deps/clock.h"
+#include "mediapipe/framework/deps/monotonic_clock.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace {
+// Tag name for clock side packet.
+constexpr char kClockTag[] = "CLOCK";
+}  // namespace
+
+// A calculator that outputs the current clock time at which it receives input
+// packets. Use a separate instance of this calculator for each input stream
+// you wish to output a clock time for.
+//
+// InputSidePacket (Optional):
+// CLOCK: A clock to use for querying the current time.
+//
+// Inputs:
+//   A single packet stream we wish to get the current clocktime for
+
+// Outputs:
+//   A single stream of absl::Time packets, representing the clock time at which
+//   we received the input stream's packets.
+
+// Example config:
+// node {
+//   calculator: "ClockTimestampCalculator"
+//   input_side_packet: "CLOCK:monotonic_clock"
+//   input_stream: "packet_stream"
+//   output_stream: "packet_clocktime_stream"
+// }
+//
+class ClockTimestampCalculator : public CalculatorBase {
+ public:
+  ClockTimestampCalculator() {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // Clock object.
+  std::shared_ptr<::mediapipe::Clock> clock_;
+};
+REGISTER_CALCULATOR(ClockTimestampCalculator);
+
+::mediapipe::Status ClockTimestampCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK_EQ(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+
+  cc->Inputs().Index(0).SetAny();
+  cc->Outputs().Index(0).Set<absl::Time>();
+
+  // Optional Clock input side packet.
+  if (cc->InputSidePackets().HasTag(kClockTag)) {
+    cc->InputSidePackets()
+        .Tag(kClockTag)
+        .Set<std::shared_ptr<::mediapipe::Clock>>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ClockTimestampCalculator::Open(CalculatorContext* cc) {
+  // Direct passthrough, as far as timestamp and bounds are concerned.
+  cc->SetOffset(TimestampDiff(0));
+
+  // Initialize the clock.
+  if (cc->InputSidePackets().HasTag(kClockTag)) {
+    clock_ = cc->InputSidePackets()
+                 .Tag("CLOCK")
+                 .Get<std::shared_ptr<::mediapipe::Clock>>();
+  } else {
+    clock_.reset(
+        ::mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ClockTimestampCalculator::Process(CalculatorContext* cc) {
+  // Push the Time packet to output.
+  auto timestamp_packet = MakePacket<absl::Time>(clock_->TimeNow());
+  cc->Outputs().Index(0).AddPacket(timestamp_packet.At(cc->InputTimestamp()));
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/detection_label_id_to_text_calculator.cc

@@ -71,16 +71,23 @@ REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator);
   const auto& options =
       cc->Options<::mediapipe::DetectionLabelIdToTextCalculatorOptions>();
 
-  std::string string_path;
+  if (options.has_label_map_path()) {
-  ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path()));
+    std::string string_path;
-  std::string label_map_string;
+    ASSIGN_OR_RETURN(string_path,
-  MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+                     PathToResourceAsFile(options.label_map_path()));
+    std::string label_map_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
 
-  std::istringstream stream(label_map_string);
+    std::istringstream stream(label_map_string);
-  std::string line;
+    std::string line;
-  int i = 0;
+    int i = 0;
-  while (std::getline(stream, line)) {
+    while (std::getline(stream, line)) {
-    label_map_[i++] = line;
+      label_map_[i++] = line;
+    }
+  } else {
+    for (int i = 0; i < options.label_size(); ++i) {
+      label_map_[i] = options.label(i);
+    }
   }
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/util/detection_label_id_to_text_calculator.proto

@@ -25,4 +25,10 @@ message DetectionLabelIdToTextCalculatorOptions {
 
   // Path to a label map file for getting the actual name of detected classes.
   optional string label_map_path = 1;
+
+  // Alternative way to specify label map
+  // label: "label for id 0"
+  // label: "label for id 1"
+  // ...
+  repeated string label = 2;
 }

mediapipe/calculators/util/detections_to_render_data_calculator.cc

@@ -27,6 +27,7 @@ namespace mediapipe {
 
 namespace {
 
+constexpr char kDetectionTag[] = "DETECTION";
 constexpr char kDetectionsTag[] = "DETECTIONS";
 constexpr char kDetectionListTag[] = "DETECTION_LIST";
 constexpr char kRenderDataTag[] = "RENDER_DATA";
@@ -62,6 +63,7 @@ constexpr float kNumScoreDecimalDigitsMultipler = 100;
 // Example config:
 // node {
 //   calculator: "DetectionsToRenderDataCalculator"
+//   input_stream: "DETECTION:detection"
 //   input_stream: "DETECTIONS:detections"
 //   input_stream: "DETECTION_LIST:detection_list"
 //   output_stream: "RENDER_DATA:render_data"
@@ -123,9 +125,13 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
 ::mediapipe::Status DetectionsToRenderDataCalculator::GetContract(
     CalculatorContract* cc) {
   RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) ||
-            cc->Inputs().HasTag(kDetectionsTag))
+            cc->Inputs().HasTag(kDetectionsTag) ||
+            cc->Inputs().HasTag(kDetectionTag))
       << "None of the input streams are provided.";
 
+  if (cc->Inputs().HasTag(kDetectionTag)) {
+    cc->Inputs().Tag(kDetectionTag).Set<Detection>();
+  }
   if (cc->Inputs().HasTag(kDetectionListTag)) {
     cc->Inputs().Tag(kDetectionListTag).Set<DetectionList>();
   }
@@ -155,8 +161,10 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
   const bool has_detection_from_vector =
       cc->Inputs().HasTag(kDetectionsTag) &&
       !cc->Inputs().Tag(kDetectionsTag).Get<std::vector<Detection>>().empty();
+  const bool has_single_detection = cc->Inputs().HasTag(kDetectionTag) &&
+                                    !cc->Inputs().Tag(kDetectionTag).IsEmpty();
   if (!options.produce_empty_packet() && !has_detection_from_list &&
-      !has_detection_from_vector) {
+      !has_detection_from_vector && !has_single_detection) {
     return ::mediapipe::OkStatus();
   }
 
@@ -176,6 +184,10 @@ REGISTER_CALCULATOR(DetectionsToRenderDataCalculator);
       AddDetectionToRenderData(detection, options, render_data.get());
     }
   }
+  if (has_single_detection) {
+    AddDetectionToRenderData(cc->Inputs().Tag(kDetectionTag).Get<Detection>(),
+                             options, render_data.get());
+  }
   cc->Outputs()
       .Tag(kRenderDataTag)
       .Add(render_data.release(), cc->InputTimestamp());

mediapipe/calculators/util/landmarks_to_detection_calculator.cc

@@ -76,7 +76,7 @@ Detection ConvertLandmarksToDetection(const NormalizedLandmarkList& landmarks) {
 // node {
 //   calculator: "LandmarksToDetectionCalculator"
 //   input_stream: "NORM_LANDMARKS:landmarks"
-//   output_stream: "DETECTIONS:detections"
+//   output_stream: "DETECTION:detections"
 // }
 class LandmarksToDetectionCalculator : public CalculatorBase {
  public:

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -186,6 +186,7 @@ class LandmarksToRenderDataCalculator : public CalculatorBase {
 
  private:
   LandmarksToRenderDataCalculatorOptions options_;
+  std::vector<int> landmark_connections_;
 };
 REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
 
@@ -217,6 +218,14 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
   cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<LandmarksToRenderDataCalculatorOptions>();
 
+  // Parse landmarks connections to a vector.
+  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
+      << "Number of entries in landmark connections must be a multiple of 2";
+
+  for (int i = 0; i < options_.landmark_connections_size(); ++i) {
+    landmark_connections_.push_back(options_.landmark_connections(i));
+  }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -236,14 +245,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     thickness *= render_scale;
   }
 
-  // Parse landmarks connections to a vector.
-  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
-      << "Number of entries in landmark connections must be a multiple of 2";
-  std::vector<int> landmark_connections;
-  for (int i = 0; i < options_.landmark_connections_size(); i += 1) {
-    landmark_connections.push_back(options_.landmark_connections(i));
-  }
-
   if (cc->Inputs().HasTag(kLandmarksTag)) {
     const LandmarkList& landmarks =
         cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
@@ -252,6 +253,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/false,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/false, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -265,15 +275,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<LandmarkList, Landmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/false,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<LandmarkList, Landmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/false, render_data.get());
-    }
   }
 
   if (cc->Inputs().HasTag(kNormLandmarksTag)) {
@@ -285,6 +286,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/true,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/true, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -298,15 +308,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/true,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/true, render_data.get());
-    }
   }
 
   cc->Outputs()

mediapipe/calculators/util/non_max_suppression_calculator.cc

@@ -303,12 +303,12 @@ class NonMaxSuppressionCalculator : public CalculatorBase {
     IndexedScores candidates;
     output_detections->clear();
     while (!remained_indexed_scores.empty()) {
+      const int original_indexed_scores_size = remained_indexed_scores.size();
       const auto& detection = detections[remained_indexed_scores[0].first];
       if (options_.min_score_threshold() > 0 &&
           detection.score(0) < options_.min_score_threshold()) {
         break;
       }
-
       remained.clear();
       candidates.clear();
       const Location location(detection.location_data());
@@ -365,8 +365,15 @@ class NonMaxSuppressionCalculator : public CalculatorBase {
           keypoint->set_y(keypoints[i * 2 + 1] / total_score);
         }
       }
-      remained_indexed_scores = std::move(remained);
+
       output_detections->push_back(weighted_detection);
+      // Breaks the loop if the size of indexed scores doesn't change after an
+      // iteration.
+      if (original_indexed_scores_size == remained.size()) {
+        break;
+      } else {
+        remained_indexed_scores = std::move(remained);
+      }
     }
   }
 

mediapipe/docs/android_archive_library.md

@@ -2,12 +2,12 @@
 
 ***Experimental Only***
 
-The MediaPipe Android archive library is a convenient way to use MediaPipe with
+The MediaPipe Android Archive (AAR) library is a convenient way to use MediaPipe
-Android Studio and Gradle. MediaPipe doesn't publish a general AAR that can be
+with Android Studio and Gradle. MediaPipe doesn't publish a general AAR that can
-used by all projects. Instead, developers need to add a mediapipe_aar() target
+be used by all projects. Instead, developers need to add a mediapipe_aar()
-to generate a custom AAR file for their own projects. This is necessary in order
+target to generate a custom AAR file for their own projects. This is necessary
-to include specific resources such as MediaPipe calculators needed for each
+in order to include specific resources such as MediaPipe calculators needed for
-project.
+each project.
 
 ### Steps to build a MediaPipe AAR
 
@@ -32,13 +32,14 @@ project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a //path/to/the/aar/build/file:aar_name
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //path/to/the/aar/build/file:aar_name
     ```
 
     For the face detection AAR target we made in the step 1, run:
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
         //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
 
     # It should print:
@@ -117,7 +118,7 @@ project.
         implementation 'com.google.code.findbugs:jsr305:3.0.2'
         implementation 'com.google.guava:guava:27.0.1-android'
         implementation 'com.google.guava:guava:27.0.1-android'
-        implementation 'com.google.protobuf:protobuf-lite:3.0.0'
+        implementation 'com.google.protobuf:protobuf-java:3.11.4''
         // CameraX core library
         def camerax_version = "1.0.0-alpha06"
         implementation "androidx.camera:camera-core:$camerax_version"

mediapipe/docs/building_examples.md

@@ -0,0 +1,327 @@
+# Building MediaPipe Examples
+
+*   [Android](#android)
+*   [iOS](#ios)
+*   [Desktop](#desktop)
+
+## Android
+
+### Prerequisite
+
+*   Java Runtime.
+*   Android SDK release 28.0.3 and above.
+*   Android NDK r18b and above.
+
+MediaPipe recommends setting up Android SDK and NDK via Android Studio (and see
+below for Android Studio setup). However, if you prefer using MediaPipe without
+Android Studio, please run
+[`setup_android_sdk_and_ndk.sh`](https://github.com/google/mediapipe/tree/master/setup_android_sdk_and_ndk.sh)
+to download and setup Android SDK and NDK before building any Android example
+apps.
+
+If Android SDK and NDK are already installed (e.g., by Android Studio), set
+$ANDROID_HOME and $ANDROID_NDK_HOME to point to the installed SDK and NDK.
+
+```bash
+export ANDROID_HOME=<path to the Android SDK>
+export ANDROID_NDK_HOME=<path to the Android NDK>
+```
+
+In order to use MediaPipe on earlier Android versions, MediaPipe needs to switch
+to a lower Android API level. You can achieve this by specifying `api_level =
+<api level integer>` in android_ndk_repository() and/or android_sdk_repository()
+in the [`WORKSPACE`](https://github.com/google/mediapipe/tree/master/WORKSPACE) file.
+
+Please verify all the necessary packages are installed.
+
+*   Android SDK Platform API Level 28 or 29
+*   Android SDK Build-Tools 28 or 29
+*   Android SDK Platform-Tools 28 or 29
+*   Android SDK Tools 26.1.1
+*   Android NDK 17c or above
+
+### Option 1: Build with Bazel in Command Line
+
+1.  To build an Android example app, for instance, for MediaPipe Hand, run:
+
+    Note: To reduce the binary size, consider appending `--linkopt="-s"` to the
+    command below to strip symbols.
+
+    ~~~
+      ```bash
+    bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu
+    ```
+    ~~~
+
+1.  Install it on a device with:
+
+    ```bash
+    adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/handtrackinggpu.apk
+    ```
+
+### Option 2: Build with Bazel in Android Studio
+
+The MediaPipe project can be imported into Android Studio using the Bazel
+plugins. This allows the MediaPipe examples to be built and modified in Android
+Studio.
+
+To incorporate MediaPipe into an existing Android Studio project, see these
+[instructions](./android_archive_library.md) that use Android Archive (AAR) and
+Gradle.
+
+The steps below use Android Studio 3.5 to build and install a MediaPipe example
+app:
+
+1.  Install and launch Android Studio 3.5.
+
+2.  Select `Configure` | `SDK Manager` | `SDK Platforms`.
+
+    *   Verify that Android SDK Platform API Level 28 or 29 is installed.
+    *   Take note of the Android SDK Location, e.g.,
+        `/usr/local/home/Android/Sdk`.
+
+3.  Select `Configure` | `SDK Manager` | `SDK Tools`.
+
+    *   Verify that Android SDK Build-Tools 28 or 29 is installed.
+    *   Verify that Android SDK Platform-Tools 28 or 29 is installed.
+    *   Verify that Android SDK Tools 26.1.1 is installed.
+    *   Verify that Android NDK 17c or above is installed.
+    *   Take note of the Android NDK Location, e.g.,
+        `/usr/local/home/Android/Sdk/ndk-bundle` or
+        `/usr/local/home/Android/Sdk/ndk/20.0.5594570`.
+
+4.  Set environment variables `$ANDROID_HOME` and `$ANDROID_NDK_HOME` to point
+    to the installed SDK and NDK.
+
+    ```bash
+    export ANDROID_HOME=/usr/local/home/Android/Sdk
+
+    # If the NDK libraries are installed by a previous version of Android Studio, do
+    export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk-bundle
+    # If the NDK libraries are installed by Android Studio 3.5, do
+    export ANDROID_NDK_HOME=/usr/local/home/Android/Sdk/ndk/<version number>
+    ```
+
+5.  Select `Configure` | `Plugins` install `Bazel`.
+
+6.  On Linux, select `File` | `Settings`| `Bazel settings`. On macos, select
+    `Android Studio` | `Preferences` | `Bazel settings`. Then, modify `Bazel
+    binary location` to be the same as the output of `$ which bazel`.
+
+7.  Select `Import Bazel Project`.
+
+    *   Select `Workspace`: `/path/to/mediapipe` and select `Next`.
+    *   Select `Generate from BUILD file`: `/path/to/mediapipe/BUILD` and select
+        `Next`.
+    *   Modify `Project View` to be the following and select `Finish`.
+
+    ```
+    directories:
+      # read project settings, e.g., .bazelrc
+      .
+      -mediapipe/objc
+      -mediapipe/examples/ios
+
+    targets:
+      //mediapipe/examples/android/...:all
+      //mediapipe/java/...:all
+
+    android_sdk_platform: android-29
+
+    sync_flags:
+      --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
+    ```
+
+8.  Select `Bazel` | `Sync` | `Sync project with Build files`.
+
+    Note: Even after doing step 4, if you still see the error: `"no such package
+    '@androidsdk//': Either the path attribute of android_sdk_repository or the
+    ANDROID_HOME environment variable must be set."`, please modify the
+    [`WORKSPACE`](https://github.com/google/mediapipe/tree/master/WORKSPACE) file to point to your
+    SDK and NDK library locations, as below:
+
+    ```
+    android_sdk_repository(
+        name = "androidsdk",
+        path = "/path/to/android/sdk"
+    )
+
+    android_ndk_repository(
+        name = "androidndk",
+        path = "/path/to/android/ndk"
+    )
+    ```
+
+9.  Connect an Android device to the workstation.
+
+10. Select `Run...` | `Edit Configurations...`.
+
+    *   Select `Templates` | `Bazel Command`.
+    *   Enter Target Expression:
+        `//mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu`
+    *   Enter Bazel command: `mobile-install`.
+    *   Enter Bazel flags: `-c opt --config=android_arm64`.
+    *   Press the `[+]` button to add the new configuration.
+    *   Select `Run` to run the example app on the connected Android device.
+
+## iOS
+
+### Prerequisite
+
+1.  Install [Xcode](https://developer.apple.com/xcode/) and the Command Line
+    Tools.
+
+    Follow Apple's instructions to obtain the required development certificates
+    and provisioning profiles for your iOS device. Install the Command Line
+    Tools by
+
+    ```bash
+    xcode-select --install
+    ```
+
+2.  Install [Bazel](https://bazel.build/).
+
+    We recommend using [Homebrew](https://brew.sh/) to get the latest version.
+
+3.  Set Python 3.7 as the default Python version and install the Python "six"
+    library.
+
+    To make Mediapipe work with TensorFlow, please set Python 3.7 as the default
+    Python version and install the Python "six" library.
+
+    ```bash
+    pip3 install --user six
+    ```
+
+4.  Clone the MediaPipe repository.
+
+    ```bash
+    git clone https://github.com/google/mediapipe.git
+    ```
+
+5.  Symlink or copy your provisioning profile to
+    `mediapipe/mediapipe/provisioning_profile.mobileprovision`.
+
+    ```bash
+    cd mediapipe
+    ln -s ~/Downloads/MyProvisioningProfile.mobileprovision mediapipe/provisioning_profile.mobileprovision
+    ```
+
+    Tip: You can use this command to see the provisioning profiles you have
+    previously downloaded using Xcode: `open
+    ~/Library/MobileDevice/"Provisioning Profiles"`. If there are none, generate
+    and download a profile on
+    [Apple's developer site](https://developer.apple.com/account/resources/).
+
+### Option 1: Build with Bazel in Command Line
+
+1.  Modify the `bundle_id` field of the app's `ios_application` target to use
+    your own identifier. For instance, for
+    [MediaPipe Hand](./hand_tracking_mobile_gpu.md), the `bundle_id` is in the
+    `HandTrackingGpuApp` target in the
+    [BUILD](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu/BUILD)
+    file.
+
+2.  Again using [MediaPipe Hand](./hand_tracking_mobile_gpu.md) for example,
+    run:
+
+    ```bash
+    bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp
+    ```
+
+    You may see a permission request from `codesign` in order to sign the app.
+
+3.  In Xcode, open the `Devices and Simulators` window (command-shift-2).
+
+4.  Make sure your device is connected. You will see a list of installed apps.
+    Press the "+" button under the list, and select the `.ipa` file built by
+    Bazel.
+
+5.  You can now run the app on your device.
+
+### Option 2: Build in Xcode
+
+Note: This workflow requires a separate tool in addition to Bazel. If it fails
+to work for some reason, please resort to the command-line build instructions in
+the previous section.
+
+1.  We will use a tool called [Tulsi](https://tulsi.bazel.build/) for generating
+    Xcode projects from Bazel build configurations.
+
+    ```bash
+    # cd out of the mediapipe directory, then:
+    git clone https://github.com/bazelbuild/tulsi.git
+    cd tulsi
+    # remove Xcode version from Tulsi's .bazelrc (see http://github.com/bazelbuild/tulsi#building-and-installing):
+    sed -i .orig '/xcode_version/d' .bazelrc
+    # build and run Tulsi:
+    sh build_and_run.sh
+    ```
+
+    This will install `Tulsi.app` inside the `Applications` directory in your
+    home directory.
+
+2.  Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app.
+
+    Important: If Tulsi displays an error saying "Bazel could not be found",
+    press the "Bazel..." button in the Packages tab and select the `bazel`
+    executable in your homebrew `/bin/` directory.
+
+3.  Select the MediaPipe config in the Configs tab, then press the Generate
+    button below. You will be asked for a location to save the Xcode project.
+    Once the project is generated, it will be opened in Xcode.
+
+4.  You can now select any of the MediaPipe demos in the target menu, and build
+    and run them as normal.
+
+    Note: When you ask Xcode to run an app, by default it will use the Debug
+    configuration. Some of our demos are computationally heavy; you may want to
+    use the Release configuration for better performance.
+
+    Tip: To switch build configuration in Xcode, click on the target menu,
+    choose "Edit Scheme...", select the Run action, and switch the Build
+    Configuration from Debug to Release. Note that this is set independently for
+    each target.
+
+## Desktop
+
+### Option 1: Running on CPU
+
+1.  To build, for example, [MediaPipe Hand](./hand_tracking_mobile_gpu.md), run:
+
+    ```bash
+    bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu
+    ```
+
+    This will open up your webcam as long as it is connected and on. Any errors
+    is likely due to your webcam being not accessible.
+
+2.  To run the application:
+
+    ```bash
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_cpu \
+      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt
+    ```
+
+### Option 2: Running on GPU
+
+Note: This currently works only on Linux, and please first follow
+[OpenGL ES Setup on Linux Desktop](./gpu.md#opengl-es-setup-on-linux-desktop).
+
+1.  To build, for example, [MediaPipe Hand](./hand_tracking_mobile_gpu.md), run:
+
+    ```bash
+    bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
+      mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
+    ```
+
+    This will open up your webcam as long as it is connected and on. Any errors
+    is likely due to your webcam being not accessible, or GPU drivers not setup
+    properly.
+
+2.  To run the application:
+
+    ```bash
+    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu \
+      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
+    ```

mediapipe/docs/examples.md

@@ -73,6 +73,18 @@ can be easily adapted to run on CPU v.s. GPU.
 *   [Android](./face_detection_mobile_cpu.md)
 *   [iOS](./face_detection_mobile_cpu.md)
 
+### Face Mesh with GPU
+
+[Face Mesh with GPU](./face_mesh_mobile_gpu.md) illustrates how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on mobile devices, utilizing GPU acceleration. The pipeline is based on
+["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724),
+and details of the underlying ML models are described in the
+[model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view).
+
+*   [Android](./face_mesh_mobile_gpu.md)
+*   [iOS](./face_mesh_mobile_gpu.md)
+
 ### Hand Detection with GPU
 
 [Hand Detection with GPU](./hand_detection_mobile_gpu.md) illustrates how to use
@@ -84,7 +96,7 @@ MediaPipe with a TFLite model for hand detection in a GPU-accelerated pipeline.
 ### Hand Tracking with GPU
 
 [Hand Tracking with GPU](./hand_tracking_mobile_gpu.md) illustrates how to use
-MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
+MediaPipe with TFLite models for hand tracking in a GPU-accelerated pipeline.
 
 *   [Android](./hand_tracking_mobile_gpu.md)
 *   [iOS](./hand_tracking_mobile_gpu.md)
@@ -92,7 +104,7 @@ MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
 ### Multi-Hand Tracking with GPU
 
 [Multi-Hand Tracking with GPU](./multi_hand_tracking_mobile_gpu.md) illustrates
-how to use MediaPipe with a TFLite model for multi-hand tracking in a
+how to use MediaPipe with TFLite models for multi-hand tracking in a
 GPU-accelerated pipeline.
 
 *   [Android](./multi_hand_tracking_mobile_gpu.md)
@@ -109,6 +121,14 @@ and model details are described in the
 
 *   [Android](./hair_segmentation_mobile_gpu.md)
 
+### Template Matching using KNIFT with CPU
+
+[Template Matching using KNIFT on Mobile](./template_matching_mobile_cpu.md)
+shows how to use MediaPipe with TFLite model for template matching using Knift
+on mobile using CPU.
+
+*   [Android](./template_matching_mobile_cpu.md)
+
 ## Desktop
 
 ### Hello World for C++
@@ -150,11 +170,19 @@ GPU with live video from a webcam.
 *   [Desktop GPU](./face_detection_desktop.md)
 *   [Desktop CPU](./face_detection_desktop.md)
 
+### Face Mesh on Desktop with Webcam
+
+[Face Mesh on Desktop with Webcam](./face_mesh_desktop.md) shows how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on desktop with webcam input.
+
+*   [Desktop GPU](./face_mesh_desktop.md)
+*   [Desktop CPU](./face_mesh_desktop.md)
 
 ### Hand Tracking on Desktop with Webcam
 
 [Hand Tracking on Desktop with Webcam](./hand_tracking_desktop.md) shows how to
-use MediaPipe with a TFLite model for hand tracking on desktop using CPU or GPU
+use MediaPipe with TFLite models for hand tracking on desktop using CPU or GPU
 with live video from a webcam.
 
 *   [Desktop GPU](./hand_tracking_desktop.md)
@@ -163,8 +191,8 @@ with live video from a webcam.
 ### Multi-Hand Tracking on Desktop with Webcam
 
 [Multi-Hand Tracking on Desktop with Webcam](./multi_hand_tracking_desktop.md)
-shows how to use MediaPipe with a TFLite model for multi-hand tracking on
+shows how to use MediaPipe with TFLite models for multi-hand tracking on desktop
-desktop using CPU or GPU with live video from a webcam.
+using CPU or GPU with live video from a webcam.
 
 *   [Desktop GPU](./multi_hand_tracking_desktop.md)
 *   [Desktop CPU](./multi_hand_tracking_desktop.md)
@@ -177,7 +205,7 @@ GPU with live video from a webcam.
 
 *   [Desktop GPU](./hair_segmentation_desktop.md)
 
-## Google Coral (machine learning acceleration with Google EdgeTPU)
+## Google Coral (ML acceleration with Google EdgeTPU)
 
 Below are code samples on how to run MediaPipe on Google Coral Dev Board.
 

mediapipe/docs/face_detection_desktop.md

@@ -14,10 +14,6 @@ We show the face detection demos with TensorFlow Lite model using the Webcam:
 
 -   [TensorFlow Lite Face Detection Demo with Webcam (GPU)](#tensorflow-lite-face-detection-demo-with-webcam-gpu)
 
-Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
-see
-[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
-
 Note: If MediaPipe depends on OpenCV 2, please see the
 [known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
 section.
@@ -46,11 +42,14 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_detection/face_de
 
 ### TensorFlow Lite Face Detection Demo with Webcam (GPU)
 
+Note: This currently works only on Linux, and please first follow
+[OpenGL ES Setup on Linux Desktop](./gpu.md#opengl-es-setup-on-linux-desktop).
+
 To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 
 ```bash
 # Video from webcam running on desktop GPU
-# This works only for linux currently
+# This works only for Linux currently
 $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/face_detection:face_detection_gpu
 
@@ -68,9 +67,6 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_detection/face_de
     --calculator_graph_config_file=mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt
 ```
 
-Issues running? Please first
-[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
-
 #### Graph
 
 ![graph visualization](images/face_detection_desktop.png)

mediapipe/docs/face_detection_mobile_cpu.md

@@ -4,6 +4,8 @@ This doc focuses on the
 [example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt)
 that performs face detection with TensorFlow Lite on CPU.
 
+![face_detection_android_gpu_gif](images/mobile/face_detection_android_gpu.gif)
+
 ## Android
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu)
@@ -19,7 +21,7 @@ adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/a
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facedetectioncpu).
 
-See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+See the general [instructions](./building_examples.md#ios) for building iOS
 examples and generating an Xcode project. This will be the FaceDetectionCpuApp
 target.
 

mediapipe/docs/face_detection_mobile_gpu.md

@@ -21,7 +21,7 @@ adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/a
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facedetectiongpu).
 
-See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+See the general [instructions](./building_examples.md#ios) for building iOS
 examples and generating an Xcode project. This will be the FaceDetectionGpuApp
 target.
 

mediapipe/docs/face_mesh_desktop.md

@@ -0,0 +1,55 @@
+## Face Mesh on Desktop with Webcam
+
+This doc focuses on running the **MediaPipe Face Mesh** pipeline to perform 3D
+face landmark estimation in real-time on desktop with webcam input. The pipeline
+internally incorporates TensorFlow Lite models. To know more about the models,
+please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+Moreover, if you are interested in running the same pipeline on Android/iOS,
+please see [Face Mesh on Android/iOS](face_mesh_mobile_gpu.md).
+
+-   [Face Mesh on Desktop with Webcam (CPU)](#face-mesh-on-desktop-with-webcam-cpu)
+
+-   [Face Mesh on Desktop with Webcam (GPU)](#face-mesh-on-desktop-with-webcam-gpu)
+
+Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+
+### Face Mesh on Desktop with Webcam (CPU)
+
+To build and run Face Mesh on desktop with webcam (CPU), run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_cpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_cpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt
+```
+
+### Face Mesh on Desktop with Webcam (GPU)
+
+Note: This currently works only on Linux, and please first follow
+[OpenGL ES Setup on Linux Desktop](./gpu.md#opengl-es-setup-on-linux-desktop).
+
+To build and run Face Mesh on desktop with webcam (GPU), run:
+
+```bash
+# This works only for Linux currently
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_gpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_gpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible, or GPU drivers not setup properly.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt
+```

mediapipe/docs/face_mesh_mobile_gpu.md

@@ -0,0 +1,90 @@
+# Face Mesh (GPU)
+
+This example focuses on running the **MediaPipe Face Mesh** pipeline on mobile
+devices to perform 3D face landmark estimation in real-time, utilizing GPU
+acceleration. The pipeline internally incorporates TensorFlow Lite models. To
+know more about the models, please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+The pipeline is related to the
+[face detection example](./face_detection_mobile_gpu.md) as it internally
+utilizes face detection and performs landmark estimation only within the
+detected region.
+
+![face_mesh_android_gpu.gif](images/mobile/face_mesh_android_gpu.gif)
+
+**MediaPipe Face Mesh** generates 468 3D face landmarks in real-time on mobile
+devices. In the visualization above, the red dots represent the landmarks, and
+the green lines connecting landmarks illustrate the contours around the eyes,
+eyebrows, lips and the entire face.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu)
+
+A prebuilt arm64 APK can be
+[downloaded here](https://drive.google.com/open?id=1pUmd7CXCL_onYMbsZo5p91cH0oNnR4gi).
+
+To build the app yourself, run:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/facemeshgpu.apk
+```
+
+## iOS
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu).
+
+See the general [instructions](./building_examples.md#ios) for building iOS
+examples and generating an Xcode project. This will be the FaceMeshGpuApp
+target.
+
+To build on the command line:
+
+```bash
+bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp
+```
+
+## Graph
+
+The face mesh [main graph](#main-graph) utilizes a
+[face landmark subgraph](#face-landmark-subgraph) from the
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
+and renders using a dedicated [face renderer subgraph](#face-renderer-subgraph).
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+![face_mesh_mobile_graph](images/mobile/face_mesh_mobile.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
+
+### Face Landmark Subgraph
+
+The
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark)
+contains several subgraphs that can be used to detect and track face landmarks.
+In particular, in this example the
+[FaceLandmarkFrontGPU](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+subgraph, suitable for images from front-facing cameras (i.e., selfie images)
+and utilizing GPU acceleration, is selected.
+
+![face_landmark_front_gpu_subgraph](images/mobile/face_landmark_front_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+
+### Face Renderer Subgraph
+
+![face_renderer_gpu_subgraph](images/mobile/face_renderer_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/subgraphs/face_renderer_gpu.pbtxt)

mediapipe/docs/gpu.md

@@ -1,13 +1,15 @@
 ## Running on GPUs
 
 -   [Overview](#overview)
--   [OpenGL Support](#opengl-support)
+-   [OpenGL ES Support](#opengl-es-support)
--   [Desktop GPUs](#desktop-gpu-linux)
+-   [Disable OpenGL ES Support](#disable-opengl-es-support)
--   [Life of a GPU calculator](#life-of-a-gpu-calculator)
+-   [OpenGL ES Setup on Linux Desktop](#opengl-es-setup-on-linux-desktop)
--   [GpuBuffer to ImageFrame converters](#gpubuffer-to-imageframe-converters)
+-   [TensorFlow CUDA Support and Setup on Linux Desktop](#tensorflow-cuda-support-and-setup-on-linux-desktop)
--   [Disable GPU support](#disable-gpu-support)
+-   [Life of a GPU Calculator](#life-of-a-gpu-calculator)
+-   [GpuBuffer to ImageFrame Converters](#gpubuffer-to-imageframe-converters)
 
 ### Overview
+
 MediaPipe supports calculator nodes for GPU compute and rendering, and allows combining multiple GPU nodes, as well as mixing them with CPU based calculator nodes. There exist several GPU APIs on mobile platforms (eg, OpenGL ES, Metal and Vulkan). MediaPipe does not attempt to offer a single cross-API GPU abstraction. Individual nodes can be written using different APIs, allowing them to take advantage of platform specific features when needed.
 
 GPU support is essential for good performance on mobile platforms, especially for real-time video. MediaPipe enables developers to write GPU compatible calculators that support the use of GPU for:
@@ -23,7 +25,7 @@ Below are the design principles for GPU support in MediaPipe
    * Because different platforms may require different techniques for best performance, the API should allow flexibility in the way things are implemented behind the scenes.
    * A calculator should be allowed maximum flexibility in using the GPU for all or part of its operation, combining it with the CPU if necessary.
 
-### OpenGL Support
+### OpenGL ES Support
 
 MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0
 on iOS. In addition, MediaPipe also supports Metal on iOS.
@@ -48,12 +50,28 @@ some Android devices. Therefore, our approach is to have one dedicated thread
 per context. Each thread issues GL commands, building up a serial command queue
 on its context, which is then executed by the GPU asynchronously.
 
-#### Desktop GPU (Linux)
+### Disable OpenGL ES Support
 
-MediaPipe GPU can run on linux systems with video cards that support OpenGL ES
+By default, building MediaPipe (with no special bazel flags) attempts to compile
-3.1 and up.
+and link against OpenGL ES (and for iOS also Metal) libraries.
 
-To check if your linux desktop GPU can run mediapipe:
+On platforms where OpenGL ES is not available (see also
+[OpenGL ES Setup on Linux Desktop](#opengl-es-setup-on-linux-desktop)), you
+should disable OpenGL ES support with:
+
+```
+$ bazel build --define MEDIAPIPE_DISABLE_GPU=1 <my-target>
+```
+
+Note: On Android and iOS, OpenGL ES is required by MediaPipe framework and the
+support should never be disabled.
+
+### OpenGL ES Setup on Linux Desktop
+
+On Linux desktop with video cards that support OpenGL ES 3.1+, MediaPipe can run
+GPU compute and rendering and perform TFLite inference on GPU.
+
+To check if your Linux desktop GPU can run MediaPipe with OpenGL ES:
 
 ```bash
 $ sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
@@ -61,7 +79,7 @@ $ sudo apt-get install mesa-utils
 $ glxinfo | grep -i opengl
 ```
 
-My linux box prints:
+For example, it may print:
 
 ```bash
 $ glxinfo | grep -i opengl
@@ -71,14 +89,133 @@ OpenGL ES profile shading language version string: OpenGL ES GLSL ES 3.20
 OpenGL ES profile extensions:
 ```
 
-*^notice the OpenGL ES 3.2 text^*
+*Notice the ES 3.20 text above.*
 
-To run MediaPipe GPU on desktop, you need to see ES 3.1 or greater printed.
+You need to see ES 3.1 or greater printed in order to perform TFLite inference
+on GPU in MediaPipe. With this setup, build with:
 
-If OpenGL ES is not printed, or is below 3.1, then the GPU inference will not
+```
-run.
+$ bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 <my-target>
+```
 
-### Life of a GPU calculator
+If only ES 3.0 or below is supported, you can still build MediaPipe targets that
+don't require TFLite inference on GPU with:
+
+```
+$ bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE <my-target>
+```
+
+Note: MEDIAPIPE_DISABLE_GL_COMPUTE is already defined automatically on all Apple
+systems (Apple doesn't support OpenGL ES 3.1+).
+
+### TensorFlow CUDA Support and Setup on Linux Desktop
+
+MediaPipe framework doesn't require CUDA for GPU compute and rendering. However,
+MediaPipe can work with TensorFlow to perform GPU inference on video cards that
+support CUDA.
+
+To enable TensorFlow GPU inference with MediaPipe, the first step is to follow
+the
+[TensorFlow GPU documentation](https://www.tensorflow.org/install/gpu#software_requirements)
+to install the required NVIDIA software on your Linux desktop.
+
+After installation, update `$PATH` and `$LD_LIBRARY_PATH` and run `ldconfig`
+with:
+
+```
+$ export PATH=/usr/local/cuda-10.1/bin${PATH:+:${PATH}}
+$ export LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64,/usr/local/cuda-10.1/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+$ sudo ldconfig
+```
+
+It's recommended to verify the installation of CUPTI, CUDA, CuDNN, and NVCC:
+
+```
+$ ls /usr/local/cuda/extras/CUPTI
+/lib64
+libcupti.so       libcupti.so.10.1.208  libnvperf_host.so        libnvperf_target.so
+libcupti.so.10.1  libcupti_static.a     libnvperf_host_static.a
+
+$ ls /usr/local/cuda-10.1
+LICENSE  bin  extras   lib64      libnvvp           nvml  samples  src      tools
+README   doc  include  libnsight  nsightee_plugins  nvvm  share    targets  version.txt
+
+$ nvcc -V
+nvcc: NVIDIA (R) Cuda compiler driver
+Copyright (c) 2005-2019 NVIDIA Corporation
+Built on Sun_Jul_28_19:07:16_PDT_2019
+Cuda compilation tools, release 10.1, V10.1.243
+
+$ ls /usr/lib/x86_64-linux-gnu/ | grep libcudnn.so
+libcudnn.so
+libcudnn.so.7
+libcudnn.so.7.6.4
+```
+
+Setting `$TF_CUDA_PATHS` is the way to declare where the CUDA library is. Note
+that the following code snippet also adds `/usr/lib/x86_64-linux-gnu` and
+`/usr/include` into `$TF_CUDA_PATHS` for cudablas and libcudnn.
+
+```
+$ export TF_CUDA_PATHS=/usr/local/cuda-10.1,/usr/lib/x86_64-linux-gnu,/usr/include
+```
+
+To make MediaPipe get TensorFlow's CUDA settings, find TensorFlow's
+[.bazelrc](https://github.com/tensorflow/tensorflow/blob/master/.bazelrc) and
+copy the `build:using_cuda` and `build:cuda` section into MediaPipe's .bazelrc
+file. For example, as of April 23, 2020, TensorFlow's CUDA setting is the
+following:
+
+```
+# This config refers to building with CUDA available. It does not necessarily
+# mean that we build CUDA op kernels.
+build:using_cuda --define=using_cuda=true
+build:using_cuda --action_env TF_NEED_CUDA=1
+build:using_cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
+
+# This config refers to building CUDA op kernels with nvcc.
+build:cuda --config=using_cuda
+build:cuda --define=using_cuda_nvcc=true
+```
+
+Finally, build MediaPipe with TensorFlow GPU with two more flags `--config=cuda`
+and `--spawn_strategy=local`. For example:
+
+```
+$ bazel build -c opt --config=cuda --spawn_strategy=local \
+    --define no_aws_support=true --copt -DMESA_EGL_NO_X11_HEADERS \
+    mediapipe/examples/desktop/object_detection:object_detection_tensorflow
+```
+
+While the binary is running, it prints out the GPU device info:
+
+```
+I external/org_tensorflow/tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
+I external/org_tensorflow/tensorflow/core/common_runtime/gpu/gpu_device.cc:1544] Found device 0 with properties: pciBusID: 0000:00:04.0 name: Tesla T4 computeCapability: 7.5 coreClock: 1.59GHz coreCount: 40 deviceMemorySize: 14.75GiB deviceMemoryBandwidth: 298.08GiB/s
+I external/org_tensorflow/tensorflow/core/common_runtime/gpu/gpu_device.cc:1686] Adding visible gpu devices: 0
+```
+
+You can monitor the GPU usage to verify whether the GPU is used for model
+inference.
+
+```
+$ nvidia-smi --query-gpu=utilization.gpu --format=csv --loop=1
+
+0 %
+0 %
+4 %
+5 %
+83 %
+21 %
+22 %
+27 %
+29 %
+100 %
+0 %
+0%
+```
+
+### Life of a GPU Calculator
 
 This section presents the basic structure of the Process method of a GPU
 calculator derived from base class GlSimpleCalculator. The GPU calculator
@@ -165,7 +302,7 @@ choices for MediaPipe GPU support:
    * Data that needs to be shared between all GPU-based calculators is provided as a external input that is implemented as a graph service and is managed by the `GlCalculatorHelper` class.
    * The combination of calculator-specific helpers and a shared graph service allows us great flexibility in managing the GPU resource: we can have a separate context per calculator, share a single context, share a lock or other synchronization primitives, etc. -- and all of this is managed by the helper and hidden from the individual calculators.
 
-### GpuBuffer to ImageFrame converters
+### GpuBuffer to ImageFrame Converters
 
 We provide two calculators called `GpuBufferToImageFrameCalculator` and `ImageFrameToGpuBufferCalculator`. These calculators convert between `ImageFrame` and `GpuBuffer`, allowing the construction of graphs that combine GPU and CPU calculators. They are supported on both iOS and Android
 
@@ -176,27 +313,3 @@ The below diagram shows the data flow in a mobile application that captures vide
 | ![How GPU calculators interact](images/gpu_example_graph.png) |
 |:--:|
 | *Video frames from the camera are fed into the graph as `GpuBuffer` packets. The input stream is accessed by two calculators in parallel. `GpuBufferToImageFrameCalculator` converts the buffer into an `ImageFrame`, which is then sent through a grayscale converter and a canny filter (both based on OpenCV and running on the CPU), whose output is then converted into a `GpuBuffer` again. A multi-input GPU calculator, GlOverlayCalculator, takes as input both the original `GpuBuffer` and the one coming out of the edge detector, and overlays them using a shader. The output is then sent back to the application using a callback calculator, and the application renders the image to the screen using OpenGL.* |
-
-### Disable GPU Support
-
-By default, building MediaPipe (with no special bazel flags) attempts to compile
-and link against OpenGL/Metal libraries.
-
-There are some command line build flags available to disable/enable GPU support
-within the MediaPipe framework:
-
-```
-# To disable *all* gpu support
-bazel build --define MEDIAPIPE_DISABLE_GPU=1  <my-target>
-
-# to enable full GPU support (OpenGL ES 3.1+ & Metal)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 <my-target>
-
-# to enable only OpenGL ES 3.0 and below (no GLES 3.1+ features)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
-```
-
-Note *MEDIAPIPE_DISABLE_GL_COMPUTE* is automatically defined on all Apple
-systems (Apple doesn't support OpenGL ES 3.1+).
-
-Note on iOS and Android, it is assumed that GPU support will be enabled.

mediapipe/docs/hair_segmentation_desktop.md

@@ -11,21 +11,20 @@ We show the hair segmentation demos with TensorFlow Lite model using the Webcam:
 
 -   [TensorFlow Lite Hair Segmentation Demo with Webcam (GPU)](#tensorflow-lite-hair-segmentation-demo-with-webcam-gpu)
 
-Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
-see
-[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
-
 Note: If MediaPipe depends on OpenCV 2, please see the
 [known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
 section.
 
 ### TensorFlow Lite Hair Segmentation Demo with Webcam (GPU)
 
+Note: This currently works only on Linux, and please first follow
+[OpenGL ES Setup on Linux Desktop](./gpu.md#opengl-es-setup-on-linux-desktop).
+
 To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 
 ```bash
 # Video from webcam running on desktop GPU
-# This works only for linux currently
+# This works only for Linux currently
 $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu
 
@@ -42,9 +41,6 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hair_segmentation/hair
     --calculator_graph_config_file=mediapipe/graphs/hair_segmentation/hair_segmentation_mobile_gpu.pbtxt
 ```
 
-Issues running? Please first
-[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
-
 #### Graph
 
 ![hair_segmentation_mobile_gpu_graph](images/mobile/hair_segmentation_mobile_gpu.png)

mediapipe/docs/hand_detection_mobile_gpu.md

@@ -41,7 +41,7 @@ adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/a
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handdetectiongpu).
 
-See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+See the general [instructions](./building_examples.md#ios) for building iOS
 examples and generating an Xcode project. This will be the HandDetectionGpuApp
 target.
 

mediapipe/docs/hand_tracking_desktop.md

@@ -13,10 +13,6 @@ We show the hand tracking demos with TensorFlow Lite model using the Webcam:
 
 -   [TensorFlow Lite Hand Tracking Demo with Webcam (GPU)](#tensorflow-lite-hand-tracking-demo-with-webcam-gpu)
 
-Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
-see
-[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
-
 Note: If MediaPipe depends on OpenCV 2, please see the
 [known issues with OpenCV 2](./object_detection_desktop.md#known-issues-with-opencv-2)
 section.
@@ -43,11 +39,14 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tra
 
 ### TensorFlow Lite Hand Tracking Demo with Webcam (GPU)
 
+Note: This currently works only on Linux, and please first follow
+[OpenGL ES Setup on Linux Desktop](./gpu.md#opengl-es-setup-on-linux-desktop).
+
 To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 
 ```bash
 # Video from webcam running on desktop GPU
-# This works only for linux currently
+# This works only for Linux currently
 $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
 
@@ -63,9 +62,6 @@ $ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tra
     --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
 ```
 
-Issues running? Please first
-[check that your GPU is supported](./gpu.md#desktop-gpu-linux)
-
 #### Graph
 
 ![graph visualization](images/hand_tracking_desktop.png)
@@ -133,6 +129,7 @@ node {
   output_stream: "LANDMARKS:hand_landmarks"
   output_stream: "NORM_RECT:hand_rect_from_landmarks"
   output_stream: "PRESENCE:hand_presence"
+  output_stream: "HANDEDNESS:handedness"
 }
 
 # Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
@@ -175,6 +172,7 @@ node {
   input_stream: "LANDMARKS:hand_landmarks"
   input_stream: "NORM_RECT:hand_rect"
   input_stream: "DETECTIONS:palm_detections"
+  input_stream: "HANDEDNESS:handedness"
   output_stream: "IMAGE:output_video"
 }
 

mediapipe/docs/hand_tracking_mobile_gpu.md

@@ -1,725 +1,154 @@
-# Hand Tracking (GPU)
+# MediaPipe Hand
 
-This doc focuses on the
+## Overview
-[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
-that performs hand tracking with TensorFlow Lite on GPU. It is related to the
-[hand detection example](./hand_detection_mobile_gpu.md), and we recommend users
-to review the hand detection example first.
 
-For overall context on hand detection and hand tracking, please read this
+The ability to perceive the shape and motion of hands can be a vital component
-[Google AI Blog post](https://mediapipe.page.link/handgoogleaiblog).
+in improving the user experience across a variety of technological domains and
+platforms. For example, it can form the basis for sign language understanding
+and hand gesture control, and can also enable the overlay of digital content and
+information on top of the physical world in augmented reality. While coming
+naturally to people, robust real-time hand perception is a decidedly challenging
+computer vision task, as hands often occlude themselves or each other (e.g.
+finger/palm occlusions and hand shakes) and lack high contrast patterns.
 
-![hand_tracking_android_gpu.gif](images/mobile/hand_tracking_android_gpu.gif)
+MediaPipe Hand is a high-fidelity hand and finger tracking solution. It employs
-
+machine learning (ML) to infer 21 3D landmarks of a hand from just a single
-In the visualization above, the red dots represent the localized hand landmarks,
+frame. Whereas current state-of-the-art approaches rely primarily on powerful
-and the green lines are simply connections between selected landmark pairs for
+desktop environments for inference, our method achieves real-time performance on
-visualization of the hand skeleton. The red box represents a hand rectangle that
+a mobile phone, and even scales to multiple hands. We hope that providing this
-covers the entire hand, derived either from hand detection (see
+hand perception functionality to the wider research and development community
-[hand detection example](./hand_detection_mobile_gpu.md)) or from the pervious
+will result in an emergence of creative use cases, stimulating new applications
-round of hand landmark localization using an ML model (see also
+and new research avenues.
-[model card](https://mediapipe.page.link/handmc)). Hand landmark localization is
-performed only within the hand rectangle for computational efficiency and
-accuracy, and hand detection is only invoked when landmark localization could
-not identify hand presence in the previous iteration.
-
-The example can also run in a mode that localizes hand landmarks in 3D (i.e.,
-estimating an extra z coordinate):
 
 ![hand_tracking_3d_android_gpu.gif](images/mobile/hand_tracking_3d_android_gpu.gif)
 
-In the visualization above, the localized hand landmarks are represented by dots
+*Fig 1. Tracked 3D hand landmarks are represented by dots in different shades,
-in different shades, with the brighter ones denoting landmarks closer to the
+with the brighter ones denoting landmarks closer to the camera.*
-camera.
+
-
+## ML Pipeline
-## Android
+
-
+MediaPipe Hand utilizes an ML pipeline consisting of multiple models working
-[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu)
+together: A palm detection model that operates on the full image and returns an
-
+oriented hand bounding box. A hand landmark model that operates on the cropped
-An arm64 APK can be
+image region defined by the palm detector and returns high-fidelity 3D hand
-[downloaded here](https://drive.google.com/open?id=1uCjS0y0O0dTDItsMh8x2cf4-l3uHW1vE),
+keypoints. This architecture is similar to that employed by our recently
-and a version running the 3D mode can be
+released [MediaPipe Face Mesh](./face_mesh_mobile_gpu.md) solution.
-[downloaded here](https://drive.google.com/open?id=1tGgzOGkcZglJO2i7e8NKSxJgVtJYS3ka).
+
-
+Providing the accurately cropped hand image to the hand landmark model
-To build the app yourself, run:
+drastically reduces the need for data augmentation (e.g. rotations, translation
-
+and scale) and instead allows the network to dedicate most of its capacity
-```bash
+towards coordinate prediction accuracy. In addition, in our pipeline the crops
-bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu
+can also be generated based on the hand landmarks identified in the previous
-```
+frame, and only when the landmark model could no longer identify hand presence
-
+is palm detection invoked to relocalize the hand.
-To build for the 3D mode, run:
+
-
+The pipeline is implemented as a MediaPipe
-```bash
+[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt),
-bazel build -c opt --config=android_arm64 --define 3D=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu
+which internally utilizes a
-```
+[palm/hand detection subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_detection_gpu.pbtxt),
-
+a
-Once the app is built, install it on Android device with:
+[hand landmark subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt)
-
+and a
-```bash
+[renderer subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt).
-adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/handtrackinggpu.apk
+For more information on how to visualize a graph and its associated subgraphs,
-```
+please see the [visualizer documentation](./visualizer.md).
-
+
-## iOS
+## Models
-
+
-[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu).
+### Palm Detection Model
-
+
-See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+To detect initial hand locations, we designed a
-examples and generating an Xcode project. This will be the HandDetectionGpuApp
+[single-shot detector](https://arxiv.org/abs/1512.02325) model optimized for
-target.
+mobile real-time uses in a manner similar to the face detection model in
-
+[MediaPipe Face Mesh](./face_mesh_mobile_gpu.md). Detecting hands is a decidedly
-To build on the command line:
+complex task: our model has to work across a variety of hand sizes with a large
-
+scale span (~20x) relative to the image frame and be able to detect occluded and
-```bash
+self-occluded hands. Whereas faces have high contrast patterns, e.g., in the eye
-bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp
+and mouth region, the lack of such features in hands makes it comparatively
-```
+difficult to detect them reliably from their visual features alone. Instead,
-
+providing additional context, like arm, body, or person features, aids accurate
-To build for the 3D mode, run:
+hand localization.
-
+
-```bash
+Our method addresses the above challenges using different strategies. First, we
-bazel build -c opt --config=ios_arm64 --define 3D=true mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp
+train a palm detector instead of a hand detector, since estimating bounding
-```
+boxes of rigid objects like palms and fists is significantly simpler than
-
+detecting hands with articulated fingers. In addition, as palms are smaller
-## Graph
+objects, the non-maximum suppression algorithm works well even for two-hand
-
+self-occlusion cases, like handshakes. Moreover, palms can be modelled using
-The hand tracking [main graph](#main-graph) internally utilizes a
+square bounding boxes (anchors in ML terminology) ignoring other aspect ratios,
-[hand detection subgraph](#hand-detection-subgraph), a
+and therefore reducing the number of anchors by a factor of 3-5. Second, an
-[hand landmark subgraph](#hand-landmark-subgraph) and a
+encoder-decoder feature extractor is used for bigger scene context awareness
-[renderer subgraph](#renderer-subgraph).
+even for small objects (similar to the RetinaNet approach). Lastly, we minimize
-
+the focal loss during training to support a large amount of anchors resulting
-The subgraphs show up in the main graph visualization as nodes colored in
+from the high scale variance.
-purple, and the subgraph itself can also be visualized just like a regular
+
-graph. For more information on how to visualize a graph that includes subgraphs,
+With the above techniques, we achieve an average precision of 95.7% in palm
-see the Visualizing Subgraphs section in the
+detection. Using a regular cross entropy loss and no decoder gives a baseline of
-[visualizer documentation](./visualizer.md).
+just 86.22%.
-
+
-### Main Graph
+### Hand Landmark Model
-
+
-![hand_tracking_mobile_graph](images/mobile/hand_tracking_mobile.png)
+After the palm detection over the whole image our subsequent hand landmark model
-
+performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt)
+the detected hand regions via regression, that is direct coordinate prediction.
-
+The model learns a consistent internal hand pose representation and is robust
-```bash
+even to partially visible hands and self-occlusions.
-# MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU.
+
-# Used in the examples in
+To obtain ground truth data, we have manually annotated ~30K real-world images
-# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
+with 21 3D coordinates, as shown below (we take Z-value from image depth map, if
-# mediapipe/examples/ios/handtrackinggpu.
+it exists per corresponding coordinate). To better cover the possible hand poses
-
+and provide additional supervision on the nature of hand geometry, we also
-# Images coming into and out of the graph.
+render a high-quality synthetic hand model over various backgrounds and map it
-input_stream: "input_video"
+to the corresponding 3D coordinates.
-output_stream: "output_video"
+
-
+![hand_crops.png](images/mobile/hand_crops.png)
-# Throttles the images flowing downstream for flow control. It passes through
+
-# the very first incoming image unaltered, and waits for downstream nodes
+*Fig 2. Top: Aligned hand crops passed to the tracking network with ground truth
-# (calculators and subgraphs) in the graph to finish their tasks before it
+annotation. Bottom: Rendered synthetic hand images with ground truth
-# passes through another image. All images that come in while waiting are
+annotation.*
-# dropped, limiting the number of in-flight images in most part of the graph to
+
-# 1. This prevents the downstream nodes from queuing up incoming images and data
+## Example Apps
-# excessively, which leads to increased latency and memory usage, unwanted in
+
-# real-time mobile applications. It also eliminates unnecessarily computation,
+Please see the [general instructions](./building_examples.md) for how to build
-# e.g., the output produced by a node may get dropped downstream if the
+MediaPipe examples for different platforms.
-# subsequent nodes are still busy processing previous inputs.
+
-node {
+#### Main Example
-  calculator: "FlowLimiterCalculator"
+
-  input_stream: "input_video"
+*   Android:
-  input_stream: "FINISHED:hand_rect"
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu),
-  input_stream_info: {
+    [Prebuilt ARM64 APK](https://drive.google.com/open?id=1uCjS0y0O0dTDItsMh8x2cf4-l3uHW1vE)
-    tag_index: "FINISHED"
+*   iOS:
-    back_edge: true
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handtrackinggpu)
-  }
+*   Desktop:
-  output_stream: "throttled_input_video"
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/hand_tracking)
-}
+
-
+#### With Multi-hand Support
-# Caches a hand-presence decision fed back from HandLandmarkSubgraph, and upon
+
-# the arrival of the next input image sends out the cached decision with the
+*   Android:
-# timestamp replaced by that of the input image, essentially generating a packet
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu),
-# that carries the previous hand-presence decision. Note that upon the arrival
+    [Prebuilt ARM64 APK](https://drive.google.com/open?id=1Wk6V9EVaz1ks_MInPqqVGvvJD01SGXDc)
-# of the very first input image, an empty packet is sent out to jump start the
+*   iOS:
-# feedback loop.
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu)
-node {
+*   Desktop:
-  calculator: "PreviousLoopbackCalculator"
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/desktop/multi_hand_tracking)
-  input_stream: "MAIN:throttled_input_video"
+
-  input_stream: "LOOP:hand_presence"
+#### Palm/Hand Detection Only (no landmarks)
-  input_stream_info: {
+
-    tag_index: "LOOP"
+*   Android:
-    back_edge: true
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectionggpu),
-  }
+    [Prebuilt ARM64 APK](https://drive.google.com/open?id=1qUlTtH7Ydg-wl_H6VVL8vueu2UCTu37E)
-  output_stream: "PREV_LOOP:prev_hand_presence"
+*   iOS:
-}
+    [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/handdetectiongpu)
-
+
-# Drops the incoming image if HandLandmarkSubgraph was able to identify hand
+## Resources
-# presence in the previous image. Otherwise, passes the incoming image through
+
-# to trigger a new round of hand detection in HandDetectionSubgraph.
+*   [Google AI Blog: On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
-node {
+*   [TensorFlow Blog: Face and hand tracking in the browser with MediaPipe and
-  calculator: "GateCalculator"
+    TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
-  input_stream: "throttled_input_video"
+*   Palm detection model:
-  input_stream: "DISALLOW:prev_hand_presence"
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
-  output_stream: "hand_detection_input_video"
+    [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
-
+*   Hand landmark model:
-  node_options: {
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
-    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+    [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-      empty_packets_as_allow: true
+*   [Model card](https://mediapipe.page.link/handmc)
-    }
-  }
-}
-
-# Subgraph that detections hands (see hand_detection_gpu.pbtxt).
-node {
-  calculator: "HandDetectionSubgraph"
-  input_stream: "hand_detection_input_video"
-  output_stream: "DETECTIONS:palm_detections"
-  output_stream: "NORM_RECT:hand_rect_from_palm_detections"
-}
-
-# Subgraph that localizes hand landmarks (see hand_landmark_gpu.pbtxt).
-node {
-  calculator: "HandLandmarkSubgraph"
-  input_stream: "IMAGE:throttled_input_video"
-  input_stream: "NORM_RECT:hand_rect"
-  output_stream: "LANDMARKS:hand_landmarks"
-  output_stream: "NORM_RECT:hand_rect_from_landmarks"
-  output_stream: "PRESENCE:hand_presence"
-}
-
-# Caches a hand rectangle fed back from HandLandmarkSubgraph, and upon the
-# arrival of the next input image sends out the cached rectangle with the
-# timestamp replaced by that of the input image, essentially generating a packet
-# that carries the previous hand rectangle. Note that upon the arrival of the
-# very first input image, an empty packet is sent out to jump start the
-# feedback loop.
-node {
-  calculator: "PreviousLoopbackCalculator"
-  input_stream: "MAIN:throttled_input_video"
-  input_stream: "LOOP:hand_rect_from_landmarks"
-  input_stream_info: {
-    tag_index: "LOOP"
-    back_edge: true
-  }
-  output_stream: "PREV_LOOP:prev_hand_rect_from_landmarks"
-}
-
-# Merges a stream of hand rectangles generated by HandDetectionSubgraph and that
-# generated by HandLandmarkSubgraph into a single output stream by selecting
-# between one of the two streams. The formal is selected if the incoming packet
-# is not empty, i.e., hand detection is performed on the current image by
-# HandDetectionSubgraph (because HandLandmarkSubgraph could not identify hand
-# presence in the previous image). Otherwise, the latter is selected, which is
-# never empty because HandLandmarkSubgraphs processes all images (that went
-# through FlowLimiterCaculator).
-node {
-  calculator: "MergeCalculator"
-  input_stream: "hand_rect_from_palm_detections"
-  input_stream: "prev_hand_rect_from_landmarks"
-  output_stream: "hand_rect"
-}
-
-# Subgraph that renders annotations and overlays them on top of the input
-# images (see renderer_gpu.pbtxt).
-node {
-  calculator: "RendererSubgraph"
-  input_stream: "IMAGE:throttled_input_video"
-  input_stream: "LANDMARKS:hand_landmarks"
-  input_stream: "NORM_RECT:hand_rect"
-  input_stream: "DETECTIONS:palm_detections"
-  output_stream: "IMAGE:output_video"
-}
-```
-
-### Hand Detection Subgraph
-
-![hand_detection_gpu_subgraph](images/mobile/hand_detection_gpu_subgraph.png)
-
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_detection_gpu.pbtxt)
-
-```bash
-# MediaPipe hand detection subgraph.
-
-type: "HandDetectionSubgraph"
-
-input_stream: "input_video"
-output_stream: "DETECTIONS:palm_detections"
-output_stream: "NORM_RECT:hand_rect_from_palm_detections"
-
-# Transforms the input image on GPU to a 256x256 image. To scale the input
-# image, the scale_mode option is set to FIT to preserve the aspect ratio,
-# resulting in potential letterboxing in the transformed image.
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "IMAGE_GPU:transformed_input_video"
-  output_stream: "LETTERBOX_PADDING:letterbox_padding"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 256
-      output_height: 256
-      scale_mode: FIT
-    }
-  }
-}
-
-# Generates a single side packet containing a TensorFlow Lite op resolver that
-# supports custom ops needed by the model used in this graph.
-node {
-  calculator: "TfLiteCustomOpResolverCalculator"
-  output_side_packet: "opresolver"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
-      use_gpu: true
-    }
-  }
-}
-
-# Converts the transformed input image on GPU into an image tensor stored as a
-# TfLiteTensor.
-node {
-  calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE_GPU:transformed_input_video"
-  output_stream: "TENSORS_GPU:image_tensor"
-}
-
-# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
-# vector of tensors representing, for instance, detection boxes/keypoints and
-# scores.
-node {
-  calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS_GPU:image_tensor"
-  output_stream: "TENSORS:detection_tensors"
-  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
-      model_path: "palm_detection.tflite"
-      use_gpu: true
-    }
-  }
-}
-
-# Generates a single side packet containing a vector of SSD anchors based on
-# the specification in the options.
-node {
-  calculator: "SsdAnchorsCalculator"
-  output_side_packet: "anchors"
-  node_options: {
-    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
-      num_layers: 5
-      min_scale: 0.1171875
-      max_scale: 0.75
-      input_size_height: 256
-      input_size_width: 256
-      anchor_offset_x: 0.5
-      anchor_offset_y: 0.5
-      strides: 8
-      strides: 16
-      strides: 32
-      strides: 32
-      strides: 32
-      aspect_ratios: 1.0
-      fixed_anchor_size: true
-    }
-  }
-}
-
-# Decodes the detection tensors generated by the TensorFlow Lite model, based on
-# the SSD anchors and the specification in the options, into a vector of
-# detections. Each detection describes a detected object.
-node {
-  calculator: "TfLiteTensorsToDetectionsCalculator"
-  input_stream: "TENSORS:detection_tensors"
-  input_side_packet: "ANCHORS:anchors"
-  output_stream: "DETECTIONS:detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
-      num_classes: 1
-      num_boxes: 2944
-      num_coords: 18
-      box_coord_offset: 0
-      keypoint_coord_offset: 4
-      num_keypoints: 7
-      num_values_per_keypoint: 2
-      sigmoid_score: true
-      score_clipping_thresh: 100.0
-      reverse_output_order: true
-
-      x_scale: 256.0
-      y_scale: 256.0
-      h_scale: 256.0
-      w_scale: 256.0
-      min_score_thresh: 0.7
-    }
-  }
-}
-
-# Performs non-max suppression to remove excessive detections.
-node {
-  calculator: "NonMaxSuppressionCalculator"
-  input_stream: "detections"
-  output_stream: "filtered_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
-      min_suppression_threshold: 0.3
-      overlap_type: INTERSECTION_OVER_UNION
-      algorithm: WEIGHTED
-      return_empty_detections: true
-    }
-  }
-}
-
-# Maps detection label IDs to the corresponding label text ("Palm"). The label
-# map is provided in the label_map_path option.
-node {
-  calculator: "DetectionLabelIdToTextCalculator"
-  input_stream: "filtered_detections"
-  output_stream: "labeled_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
-      label_map_path: "palm_detection_labelmap.txt"
-    }
-  }
-}
-
-# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
-# letterboxed image (after image transformation with the FIT scale mode) to the
-# corresponding locations on the same image with the letterbox removed (the
-# input image to the graph before image transformation).
-node {
-  calculator: "DetectionLetterboxRemovalCalculator"
-  input_stream: "DETECTIONS:labeled_detections"
-  input_stream: "LETTERBOX_PADDING:letterbox_padding"
-  output_stream: "DETECTIONS:palm_detections"
-}
-
-# Extracts image size from the input images.
-node {
-  calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "SIZE:image_size"
-}
-
-# Converts results of palm detection into a rectangle (normalized by image size)
-# that encloses the palm and is rotated such that the line connecting center of
-# the wrist and MCP of the middle finger is aligned with the Y-axis of the
-# rectangle.
-node {
-  calculator: "DetectionsToRectsCalculator"
-  input_stream: "DETECTIONS:palm_detections"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_RECT:palm_rect"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
-      rotation_vector_start_keypoint_index: 0  # Center of wrist.
-      rotation_vector_end_keypoint_index: 2  # MCP of middle finger.
-      rotation_vector_target_angle_degrees: 90
-      output_zero_rect_for_empty_detections: true
-    }
-  }
-}
-
-# Expands and shifts the rectangle that contains the palm so that it's likely
-# to cover the entire hand.
-node {
-  calculator: "RectTransformationCalculator"
-  input_stream: "NORM_RECT:palm_rect"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "hand_rect_from_palm_detections"
-  node_options: {
-    [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 2.6
-      scale_y: 2.6
-      shift_y: -0.5
-      square_long: true
-    }
-  }
-}
-```
-
-### Hand Landmark Subgraph
-
-![hand_landmark_gpu_subgraph.pbtxt](images/mobile/hand_landmark_gpu_subgraph.png)
-
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt)
-
-```bash
-# MediaPipe hand landmark localization subgraph.
-
-type: "HandLandmarkSubgraph"
-
-input_stream: "IMAGE:input_video"
-input_stream: "NORM_RECT:hand_rect"
-output_stream: "LANDMARKS:hand_landmarks"
-output_stream: "NORM_RECT:hand_rect_for_next_frame"
-output_stream: "PRESENCE:hand_presence"
-
-# Crops the rectangle that contains a hand from the input image.
-node {
-  calculator: "ImageCroppingCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  input_stream: "NORM_RECT:hand_rect"
-  output_stream: "IMAGE_GPU:hand_image"
-}
-
-# Transforms the input image on GPU to a 256x256 image. To scale the input
-# image, the scale_mode option is set to FIT to preserve the aspect ratio,
-# resulting in potential letterboxing in the transformed image.
-node: {
-  calculator: "ImageTransformationCalculator"
-  input_stream: "IMAGE_GPU:hand_image"
-  output_stream: "IMAGE_GPU:transformed_hand_image"
-  output_stream: "LETTERBOX_PADDING:letterbox_padding"
-  node_options: {
-    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
-      output_width: 256
-      output_height: 256
-      scale_mode: FIT
-    }
-  }
-}
-
-# Converts the transformed input image on GPU into an image tensor stored as a
-# TfLiteTensor.
-node {
-  calculator: "TfLiteConverterCalculator"
-  input_stream: "IMAGE_GPU:transformed_hand_image"
-  output_stream: "TENSORS_GPU:image_tensor"
-}
-
-# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
-# vector of tensors representing, for instance, detection boxes/keypoints and
-# scores.
-node {
-  calculator: "TfLiteInferenceCalculator"
-  input_stream: "TENSORS_GPU:image_tensor"
-  output_stream: "TENSORS:output_tensors"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
-      model_path: "hand_landmark.tflite"
-      use_gpu: true
-    }
-  }
-}
-
-# Splits a vector of tensors into multiple vectors.
-node {
-  calculator: "SplitTfLiteTensorVectorCalculator"
-  input_stream: "output_tensors"
-  output_stream: "landmark_tensors"
-  output_stream: "hand_flag_tensor"
-  node_options: {
-    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
-      ranges: { begin: 0 end: 1 }
-      ranges: { begin: 1 end: 2 }
-    }
-  }
-}
-
-# Converts the hand-flag tensor into a float that represents the confidence
-# score of hand presence.
-node {
-  calculator: "TfLiteTensorsToFloatsCalculator"
-  input_stream: "TENSORS:hand_flag_tensor"
-  output_stream: "FLOAT:hand_presence_score"
-}
-
-# Applies a threshold to the confidence score to determine whether a hand is
-# present.
-node {
-  calculator: "ThresholdingCalculator"
-  input_stream: "FLOAT:hand_presence_score"
-  output_stream: "FLAG:hand_presence"
-  node_options: {
-    [type.googleapis.com/mediapipe.ThresholdingCalculatorOptions] {
-      threshold: 0.1
-    }
-  }
-}
-
-# Decodes the landmark tensors into a vector of lanmarks, where the landmark
-# coordinates are normalized by the size of the input image to the model.
-node {
-  calculator: "TfLiteTensorsToLandmarksCalculator"
-  input_stream: "TENSORS:landmark_tensors"
-  output_stream: "NORM_LANDMARKS:landmarks"
-  node_options: {
-    [type.googleapis.com/mediapipe.TfLiteTensorsToLandmarksCalculatorOptions] {
-      num_landmarks: 21
-      input_image_width: 256
-      input_image_height: 256
-    }
-  }
-}
-
-# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
-# image (after image transformation with the FIT scale mode) to the
-# corresponding locations on the same image with the letterbox removed (hand
-# image before image transformation).
-node {
-  calculator: "LandmarkLetterboxRemovalCalculator"
-  input_stream: "LANDMARKS:landmarks"
-  input_stream: "LETTERBOX_PADDING:letterbox_padding"
-  output_stream: "LANDMARKS:scaled_landmarks"
-}
-
-# Projects the landmarks from the cropped hand image to the corresponding
-# locations on the full image before cropping (input to the graph).
-node {
-  calculator: "LandmarkProjectionCalculator"
-  input_stream: "NORM_LANDMARKS:scaled_landmarks"
-  input_stream: "NORM_RECT:hand_rect"
-  output_stream: "NORM_LANDMARKS:hand_landmarks"
-}
-
-# Extracts image size from the input images.
-node {
-  calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE_GPU:input_video"
-  output_stream: "SIZE:image_size"
-}
-
-# Converts hand landmarks to a detection that tightly encloses all landmarks.
-node {
-  calculator: "LandmarksToDetectionCalculator"
-  input_stream: "NORM_LANDMARKS:hand_landmarks"
-  output_stream: "DETECTION:hand_detection"
-}
-
-# Converts the hand detection into a rectangle (normalized by image size)
-# that encloses the hand and is rotated such that the line connecting center of
-# the wrist and MCP of the middle finger is aligned with the Y-axis of the
-# rectangle.
-node {
-  calculator: "DetectionsToRectsCalculator"
-  input_stream: "DETECTION:hand_detection"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "NORM_RECT:hand_rect_from_landmarks"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
-      rotation_vector_start_keypoint_index: 0  # Center of wrist.
-      rotation_vector_end_keypoint_index: 9  # MCP of middle finger.
-      rotation_vector_target_angle_degrees: 90
-    }
-  }
-}
-
-# Expands the hand rectangle so that in the next video frame it's likely to
-# still contain the hand even with some motion.
-node {
-  calculator: "RectTransformationCalculator"
-  input_stream: "NORM_RECT:hand_rect_from_landmarks"
-  input_stream: "IMAGE_SIZE:image_size"
-  output_stream: "hand_rect_for_next_frame"
-  node_options: {
-    [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
-      scale_x: 1.6
-      scale_y: 1.6
-      square_long: true
-    }
-  }
-}
-```
-
-### Renderer Subgraph
-
-![hand_renderer_gpu_subgraph.pbtxt](images/mobile/hand_renderer_gpu_subgraph.png)
-
-[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt)
-
-```bash
-# MediaPipe hand tracking rendering subgraph.
-
-type: "RendererSubgraph"
-
-input_stream: "IMAGE:input_image"
-input_stream: "DETECTIONS:detections"
-input_stream: "LANDMARKS:landmarks"
-input_stream: "NORM_RECT:rect"
-output_stream: "IMAGE:output_image"
-
-# Converts detections to drawing primitives for annotation overlay.
-node {
-  calculator: "DetectionsToRenderDataCalculator"
-  input_stream: "DETECTIONS:detections"
-  output_stream: "RENDER_DATA:detection_render_data"
-  node_options: {
-    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
-      thickness: 4.0
-      color { r: 0 g: 255 b: 0 }
-    }
-  }
-}
-
-# Converts landmarks to drawing primitives for annotation overlay.
-node {
-  calculator: "LandmarksToRenderDataCalculator"
-  input_stream: "NORM_LANDMARKS:landmarks"
-  output_stream: "RENDER_DATA:landmark_render_data"
-  node_options: {
-    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
-      landmark_connections: 0
-      landmark_connections: 1
-      landmark_connections: 1
-      landmark_connections: 2
-      landmark_connections: 2
-      landmark_connections: 3
-      landmark_connections: 3
-      landmark_connections: 4
-      landmark_connections: 0
-      landmark_connections: 5
-      landmark_connections: 5
-      landmark_connections: 6
-      landmark_connections: 6
-      landmark_connections: 7
-      landmark_connections: 7
-      landmark_connections: 8
-      landmark_connections: 5
-      landmark_connections: 9
-      landmark_connections: 9
-      landmark_connections: 10
-      landmark_connections: 10
-      landmark_connections: 11
-      landmark_connections: 11
-      landmark_connections: 12
-      landmark_connections: 9
-      landmark_connections: 13
-      landmark_connections: 13
-      landmark_connections: 14
-      landmark_connections: 14
-      landmark_connections: 15
-      landmark_connections: 15
-      landmark_connections: 16
-      landmark_connections: 13
-      landmark_connections: 17
-      landmark_connections: 0
-      landmark_connections: 17
-      landmark_connections: 17
-      landmark_connections: 18
-      landmark_connections: 18
-      landmark_connections: 19
-      landmark_connections: 19
-      landmark_connections: 20
-      landmark_color { r: 255 g: 0 b: 0 }
-      connection_color { r: 0 g: 255 b: 0 }
-      thickness: 4.0
-    }
-  }
-}
-
-# Converts normalized rects to drawing primitives for annotation overlay.
-node {
-  calculator: "RectToRenderDataCalculator"
-  input_stream: "NORM_RECT:rect"
-  output_stream: "RENDER_DATA:rect_render_data"
-  node_options: {
-    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
-      filled: false
-      color { r: 255 g: 0 b: 0 }
-      thickness: 4.0
-    }
-  }
-}
-
-# Draws annotations and overlays them on top of the input images.
-node {
-  calculator: "AnnotationOverlayCalculator"
-  input_stream: "IMAGE_GPU:input_image"
-  input_stream: "detection_render_data"
-  input_stream: "landmark_render_data"
-  input_stream: "rect_render_data"
-  output_stream: "IMAGE_GPU:output_image"
-}
-```

mediapipe/docs/hello_world_android.md

@@ -32,7 +32,7 @@ We will be using the following graph, [`edge_detection_mobile_gpu.pbtxt`]:
 ```
 # MediaPipe graph that performs GPU Sobel edge detection on a live video stream.
 # Used in the examples
-# mediapipe/examples/android/src/java/com/mediapipe/apps/edgedetectiongpu.
+# mediapipe/examples/android/src/java/com/mediapipe/apps/basic.
 # mediapipe/examples/ios/edgedetectiongpu.
 
 # Images coming into and out of the graph.
@@ -80,15 +80,15 @@ applications using `bazel`.
 
 Create a new directory where you will create your Android application. For
 example, the complete code of this tutorial can be found at
-`mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu`.
+`mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic`. We
-We will refer to this path as `$APPLICATION_PATH` throughout the codelab.
+will refer to this path as `$APPLICATION_PATH` throughout the codelab.
 
 Note that in the path to the application:
 
-*   The application is named `edgedetectiongpu`.
+*   The application is named `helloworld`.
 *   The `$PACKAGE_PATH` of the application is
-    `com.google.mediapipe.apps.edgdetectiongpu`. This is used in code snippets in
+    `com.google.mediapipe.apps.basic`. This is used in code snippets in this
-    this tutorial, so please remember to use your own `$PACKAGE_PATH` when you
+    tutorial, so please remember to use your own `$PACKAGE_PATH` when you
     copy/use the code snippets.
 
 Add a file `activity_main.xml` to `$APPLICATION_PATH/res/layout`. This displays
@@ -119,7 +119,7 @@ Add a simple `MainActivity.java` to `$APPLICATION_PATH` which loads the content
 of the `activity_main.xml` layout as shown below:
 
 ```
-package com.google.mediapipe.apps.edgedetectiongpu;
+package com.google.mediapipe.apps.basic;
 
 import android.os.Bundle;
 import androidx.appcompat.app.AppCompatActivity;
@@ -141,7 +141,7 @@ launches `MainActivity` on application start:
 ```
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    package="com.google.mediapipe.apps.edgedetectiongpu">
+    package="com.google.mediapipe.apps.basic">
 
   <uses-sdk
       android:minSdkVersion="19"
@@ -149,11 +149,11 @@ launches `MainActivity` on application start:
 
   <application
       android:allowBackup="true"
-      android:label="@string/app_name"
+      android:label="${appName}"
       android:supportsRtl="true"
       android:theme="@style/AppTheme">
       <activity
-          android:name=".MainActivity"
+          android:name="${mainActivity}"
           android:exported="true"
           android:screenOrientation="portrait">
           <intent-filter>
@@ -166,17 +166,8 @@ launches `MainActivity` on application start:
 </manifest>
 ```
 
-To get `@string/app_name`, we need to add a file `strings.xml` to
+In our application we are using a `Theme.AppCompat` theme in the app, so we need
-`$APPLICATION_PATH/res/values/`:
+appropriate theme references. Add `colors.xml` to
-
-```
-<resources>
-    <string name="app_name" translatable="false">Edge Detection GPU</string>
-</resources>
-```
-
-Also, in our application we are using a `Theme.AppCompat` theme in the app, so
-we need appropriate theme references. Add `colors.xml` to
 `$APPLICATION_PATH/res/values/`:
 
 ```
@@ -204,11 +195,13 @@ Add `styles.xml` to `$APPLICATION_PATH/res/values/`:
 </resources>
 ```
 
-To build the application, add a `BUILD` file to `$APPLICATION_PATH`:
+To build the application, add a `BUILD` file to `$APPLICATION_PATH`, and
+`${appName}` and `${mainActivity}` in the manifest will be replaced by strings
+specified in `BUILD` as shown below.
 
 ```
 android_library(
-    name = "mediapipe_lib",
+    name = "basic_lib",
     srcs = glob(["*.java"]),
     manifest = "AndroidManifest.xml",
     resource_files = glob(["res/**"]),
@@ -219,34 +212,36 @@ android_library(
 )
 
 android_binary(
-    name = "edgedetectiongpu",
+    name = "helloworld",
-    aapt_version = "aapt2",
     manifest = "AndroidManifest.xml",
-    manifest_values = {"applicationId": "com.google.mediapipe.apps.edgedetectiongpu"},
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.apps.basic",
+        "appName": "Hello World",
+        "mainActivity": ".MainActivity",
+    },
     multidex = "native",
     deps = [
-        ":mediapipe_lib",
+        ":basic_lib",
     ],
 )
-
 ```
 
 The `android_library` rule adds dependencies for `MainActivity`, resource files
 and `AndroidManifest.xml`.
 
-The `android_binary` rule, uses the `mediapipe_lib` Android library generated to
+The `android_binary` rule, uses the `basic_lib` Android library generated to
 build a binary APK for installation on your Android device.
 
 To build the app, use the following command:
 
 ```
-bazel build -c opt --config=android_arm64 $APPLICATION_PATH
+bazel build -c opt --config=android_arm64 $APPLICATION_PATH:helloworld
 ```
 
 Install the generated APK file using `adb install`. For example:
 
 ```
-adb install bazel-bin/$APPLICATION_PATH/edgedetectiongpu.apk
+adb install bazel-bin/$APPLICATION_PATH/helloworld.apk
 ```
 
 Open the application on your device. It should display a screen with the text
@@ -438,22 +433,58 @@ visible so that we can start seeing frames from the `previewFrameTexture`.
 
 However, before starting the camera, we need to decide which camera we want to
 use. [`CameraXPreviewHelper`] inherits from [`CameraHelper`] which provides two
-options, `FRONT` and `BACK`. We will use `BACK` camera for this application to
+options, `FRONT` and `BACK`. We can pass in the decision from the `BUILD` file
-perform edge detection on a live scene that we view from the camera.
+as metadata such that no code change is required to build a another version of
+the app using a different camera.
 
-Add the following line to define `CAMERA_FACING` for our application,
+Assuming we want to use `BACK` camera to perform edge detection on a live scene
+that we view from the camera, add the metadata into `AndroidManifest.xml`:
 
 ```
-private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.BACK;
+      ...
+      <meta-data android:name="cameraFacingFront" android:value="${cameraFacingFront}"/>
+  </application>
+</manifest>
 ```
 
-`CAMERA_FACING` is a static variable as we will use the same camera throughout
+and specify the selection in `BUILD` in the `helloworld` android binary rule
-the application from start to finish.
+with a new entry in `manifest_values`:
+
+```
+manifest_values = {
+    "applicationId": "com.google.mediapipe.apps.basic",
+    "appName": "Hello World",
+    "mainActivity": ".MainActivity",
+    "cameraFacingFront": "False",
+},
+```
+
+Now, in `MainActivity` to retrieve the metadata specified in `manifest_values`,
+add an [`ApplicationInfo`] object:
+
+```
+private ApplicationInfo applicationInfo;
+```
+
+In the `onCreate()` function, add:
+
+```
+try {
+  applicationInfo =
+      getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA);
+} catch (NameNotFoundException e) {
+  Log.e(TAG, "Cannot find application info: " + e);
+}
+```
 
 Now add the following line at the end of the `startCamera()` function:
 
 ```
-cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+CameraHelper.CameraFacing cameraFacing =
+    applicationInfo.metaData.getBoolean("cameraFacingFront", false)
+        ? CameraHelper.CameraFacing.FRONT
+        : CameraHelper.CameraFacing.BACK;
+cameraHelper.startCamera(this, cameraFacing, /*surfaceTexture=*/ null);
 ```
 
 At this point, the application should build successfully. However, when you run
@@ -595,24 +626,13 @@ build rule:
 
 MediaPipe graphs are `.pbtxt` files, but to use them in the application, we need
 to use the `mediapipe_binary_graph` build rule to generate a `.binarypb` file.
-We can then use an application specific alias for the graph via the `genrule`
-build rule. Add the following `genrule` to use an alias for the edge detection
-graph:
 
-```
+In the `helloworld` android binary build rule, add the `mediapipe_binary_graph`
-genrule(
+target specific to the graph as an asset:
-    name = "binary_graph",
-    srcs = ["//mediapipe/graphs/edge_detection:mobile_gpu_binary_graph"],
-    outs = ["edgedetectiongpu.binarypb"],
-    cmd = "cp $< $@",
-)
-```
-
-Then in the `mediapipe_lib` build rule, add assets:
 
 ```
 assets = [
-  ":binary_graph",
+  "//mediapipe/graphs/edge_detection:mobile_gpu_binary_graph",
 ],
 assets_dir = "",
 ```
@@ -620,6 +640,26 @@ assets_dir = "",
 In the `assets` build rule, you can also add other assets such as TensorFlowLite
 models used in your graph.
 
+In addition, add additional `manifest_values` for properties specific to the
+graph, to be later retrieved in `MainActivity`:
+
+```
+manifest_values = {
+    "applicationId": "com.google.mediapipe.apps.basic",
+    "appName": "Hello World",
+    "mainActivity": ".MainActivity",
+    "cameraFacingFront": "False",
+    "binaryGraphName": "mobile_gpu.binarypb",
+    "inputVideoStreamName": "input_video",
+    "outputVideoStreamName": "output_video",
+},
+```
+
+Note that `binaryGraphName` indicates the filename of the binary graph,
+determined by the `output_name` field in the `mediapipe_binary_graph` target.
+`inputVideoStreamName` and `outputVideoStreamName` are the input and output
+video stream name specified in the graph respectively.
+
 Now, the `MainActivity` needs to load the MediaPipe framework. Also, the
 framework uses OpenCV, so `MainActvity` should also load `OpenCV`. Use the
 following code in `MainActivity` (inside the class, but not inside any function)
@@ -648,15 +688,6 @@ Initialize the asset manager in `onCreate(Bundle)` before initializing
 AndroidAssetUtil.initializeNativeAssetManager(this);
 ```
 
-Declare a static variable with the graph name, the name of the input stream and
-the name of the output stream:
-
-```
-private static final String BINARY_GRAPH_NAME = "edgedetectiongpu.binarypb";
-private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
-private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
-```
-
 Now, we need to setup a [`FrameProcessor`] object that sends camera frames
 prepared by the `converter` to the MediaPipe graph and runs the graph, prepares
 the output and then updates the `previewDisplayView` to display the output. Add
@@ -673,9 +704,9 @@ processor =
     new FrameProcessor(
         this,
         eglManager.getNativeContext(),
-        BINARY_GRAPH_NAME,
+        applicationInfo.metaData.getString("binaryGraphName"),
-        INPUT_VIDEO_STREAM_NAME,
+        applicationInfo.metaData.getString("inputVideoStreamName"),
-        OUTPUT_VIDEO_STREAM_NAME);
+        applicationInfo.metaData.getString("outputVideoStreamName"));
 ```
 
 The `processor` needs to consume the converted frames from the `converter` for
@@ -712,8 +743,9 @@ feed! Congrats!
 ![edge_detection_android_gpu_gif](images/mobile/edge_detection_android_gpu.gif)
 
 If you ran into any issues, please see the full code of the tutorial
-[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu).
+[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic).
 
+[`ApplicationInfo`]:https://developer.android.com/reference/android/content/pm/ApplicationInfo
 [`AndroidAssetUtil`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/framework/AndroidAssetUtil.java
 [Bazel]:https://bazel.build/
 [`CameraHelper`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/CameraHelper.java
@@ -721,7 +753,6 @@ If you ran into any issues, please see the full code of the tutorial
 [`CameraXPreviewHelper`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/CameraXPreviewHelper.java
 [developer options]:https://developer.android.com/studio/debug/dev-options
 [`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt
-[`EdgeDetectionGPU` example]:https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/
 [`EglManager`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/glutil/EglManager.java
 [`ExternalTextureConverter`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
 [`FrameLayout`]:https://developer.android.com/reference/android/widget/FrameLayout

mediapipe/docs/hello_world_ios.md

@@ -183,7 +183,7 @@ bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/edgedetectiongpu:Ed
 
 Then, go back to XCode, open Window > Devices and Simulators, select your
 device, and add the `.ipa` file generated by the command above to your device.
-Here is the document on [setting up and compiling](./mediapipe_ios_setup.md) iOS
+Here is the document on [setting up and compiling](./building_examples.md#ios) iOS
 MediaPipe apps.
 
 Open the application on your device. Since it is empty, it should display a
@@ -348,7 +348,7 @@ responded. Add the following code to `viewWillAppear:animated`:
 ```
 [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
   if (granted) {
-    dispatch_queue(_videoQueue, ^{
+    dispatch_async(_videoQueue, ^{
       [_cameraSource start];
     });
   }
@@ -405,7 +405,7 @@ Declare a static constant with the name of the graph, the input stream and the
 output stream:
 
 ```
-static NSString* const kGraphName = @"android_gpu";
+static NSString* const kGraphName = @"mobile_gpu";
 
 static const char* kInputStream = "input_video";
 static const char* kOutputStream = "output_video";
@@ -483,7 +483,7 @@ in our app:
       NSLog(@"Failed to start graph: %@", error);
     }
 
-    dispatch_queue(_videoQueue, ^{
+    dispatch_async(_videoQueue, ^{
       [_cameraSource start];
     });
   }

mediapipe/docs/how_to_questions.md

@@ -16,7 +16,7 @@ type [`ImageFrame`] and [`GpuBuffer`]. [`ImageFrame`] refers to image data in
 CPU memory in any of a number of bitmap image formats. [`GpuBuffer`] refers to
 image data in GPU memory. You can find more detail in the Framework Concepts
 section
-[GpuBuffer to ImageFrame converters](./gpu.md).
+[GpuBuffer to ImageFrame Converters](./gpu.md#gpubuffer-to-imageframe-converters).
 You can see an example in:
 
 *   [`object_detection_mobile_cpu.pbtxt`]