Project import generated by Copybara.

GitOrigin-RevId: d0039a576e2db9c0fcefffd26a527df74cbe145b

.bazelrc

@@ -12,9 +12,9 @@ build --copt='-Wno-comment'
 build --copt='-Wno-return-type'
 build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
-# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
+
-build --incompatible_disable_deprecated_attr_params=false
+# Tensorflow needs remote repo
-build --incompatible_depset_is_not_iterable=false
+build --experimental_repo_remote_exec
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
@@ -23,6 +23,10 @@ build --apple_platform_type=macos
 build --apple_generate_dsym
 
 # Android configs.
+# Note: the documentation tells us to use @androidndk//:default_crosstool, but
+# the automatic configuration transition uses //external:android/crosstool.
+# Using it here works and spares us from having two different config_settings
+# for Android.
 build:android --crosstool_top=//external:android/crosstool
 build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
 build:android --linkopt=-landroid

Dockerfile

@@ -48,7 +48,7 @@ RUN pip install future
 RUN pip3 install six
 
 # Install bazel
-ARG BAZEL_VERSION=1.1.0
+ARG BAZEL_VERSION=2.0.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

README.md

@@ -9,26 +9,34 @@
 
 ## ML Solutions in MediaPipe
 
-* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html)
+* [Face Mesh](mediapipe/docs/face_mesh_mobile_gpu.md)
+* [Hand Detection](mediapipe/docs/hand_detection_mobile_gpu.md)
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 * [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
-* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [(web demo)](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
-* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md) [[Web Demo]](https://viz.mediapipe.dev/runner/demos/hair_segmentation/hair_segmentation.html)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 * [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
 * [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
-* [AutoFlip](mediapipe/docs/autoflip.md)
+* [AutoFlip: Intelligent Video Reframing](mediapipe/docs/autoflip.md)
+* [KNIFT: Template Matching with Neural Image Features](mediapipe/docs/template_matching_mobile_cpu.md)
 
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
-![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
+![face_mesh](mediapipe/docs/images/mobile/face_mesh_android_gpu_small.gif)
-![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
+![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
 ![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
+![objectron_shoes](mediapipe/docs/images/mobile/objectron_shoe_android_gpu_small.gif)
+![objectron_chair](mediapipe/docs/images/mobile/objectron_chair_android_gpu_small.gif)
+![template_matching](mediapipe/docs/images/mobile/template_matching_android_cpu_small.gif)
 
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).
 
 ## Getting started
-See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
+See mobile, desktop, web and Google Coral [examples](mediapipe/docs/examples.md).
 
 Check out some web demos [[Edge detection]](https://viz.mediapipe.dev/runner/demos/edge_detection/edge_detection.html) [[Face detection]](https://viz.mediapipe.dev/runner/demos/face_detection/face_detection.html) [[Hand Tracking]](https://viz.mediapipe.dev/runner/demos/hand_tracking/hand_tracking.html)
 
@@ -40,10 +48,15 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 ## Visualizing MediaPipe graphs
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
+## Google Open Source Code search
+Search MediaPipe Github repository using [Google Open Source code search](https://t.co/LSZnbMUUnT?amp=1)
+
 ## Videos
 *  [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
 
 ## Publications
+* [MediaPipe KNIFT: Template-based Feature Matching](https://mediapipe.page.link/knift-blog)
+* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
 * [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
 * [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)
 * [Google Developer Blog: MediaPipe on the Web](https://mediapipe.page.link/webdevblog)
@@ -52,6 +65,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
+* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
 * [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
 * [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
 * [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)

WORKSPACE

@@ -2,16 +2,15 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.8.0"
+skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
+    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
-    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
+    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "1.0.0",
+versions.check(minimum_bazel_version = "2.0.0")
-               maximum_bazel_version = "1.2.1")
 
 
 # ABSL cpp library lts_2020_02_25
@@ -93,11 +92,13 @@ http_archive(
     build_file = "@//third_party:libyuv.BUILD",
 )
 
+# Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download.
+# ...but the Java download is currently broken, so we use the "source" download.
 http_archive(
     name = "com_google_protobuf_javalite",
-    sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
+    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
+    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
 )
 
 http_archive(
@@ -117,23 +118,18 @@ http_archive(
     ],
 )
 
-# 2020-02-12
+# 2020-04-01
-# The last commit before TensorFlow switched to Bazel 2.0
+_TENSORFLOW_GIT_COMMIT = "805e47cea96c7e8c6fccf494d40a2392dc99fdd8"
-_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
+_TENSORFLOW_SHA256= "9ee3ae604c2e1345ac60345becee6d659364721513f9cb8652eb2e7138320ca5"
-_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
 http_archive(
     name = "org_tensorflow",
     urls = [
       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
-    # A compatibility patch
     patches = [
-        "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff",
+        "@//third_party:org_tensorflow_compatibility_fixes.diff",
-        # Updates for XNNPACK: https://github.com/tensorflow/tensorflow/commit/cfc31e324c8de6b52f752a39cb161d99d853ca99
+        "@//third_party:org_tensorflow_protobuf_updates.diff",
-        "@//third_party:org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff",
-        # CpuInfo's build rule fixes.
-        "@//third_party:org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff",
     ],
     patch_args = [
         "-p1",
@@ -158,10 +154,6 @@ http_archive(
     sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
 )
 
-# Please run
-# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
-#                        libopencv-calib3d-dev libopencv-features2d-dev \
-#                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
     build_file = "@//third_party:opencv_linux.BUILD",
@@ -174,7 +166,6 @@ new_local_repository(
     path = "/usr"
 )
 
-# Please run $ brew install opencv@3
 new_local_repository(
     name = "macos_opencv",
     build_file = "@//third_party:opencv_macos.BUILD",
@@ -207,79 +198,6 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
-RULES_JVM_EXTERNAL_TAG = "2.2"
-RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
-
-http_archive(
-    name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
-    sha256 = RULES_JVM_EXTERNAL_SHA,
-    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
-)
-
-load("@rules_jvm_external//:defs.bzl", "maven_install")
-
-maven_install(
-    artifacts = [
-        "androidx.annotation:annotation:aar:1.1.0",
-        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
-        "androidx.camera:camera-core:aar:1.0.0-alpha06",
-        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
-        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
-        "androidx.core:core:aar:1.1.0-rc03",
-        "androidx.legacy:legacy-support-v4:aar:1.0.0",
-        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
-        "com.google.android.material:material:aar:1.0.0-rc01",
-    ],
-    repositories = [
-        "https://dl.google.com/dl/android/maven2",
-        "https://repo1.maven.org/maven2",
-    ],
-)
-
-maven_server(
-    name = "google_server",
-    url = "https://dl.google.com/dl/android/maven2",
-)
-
-maven_jar(
-    name = "androidx_lifecycle",
-    artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
-    sha1 = "e070ffae07452331bc5684734fce6831d531785c",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "androidx_concurrent_futures",
-    artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
-    sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "com_google_guava_android",
-    artifact = "com.google.guava:guava:27.0.1-android",
-    sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
-)
-
-maven_jar(
-    name = "com_google_common_flogger",
-    artifact = "com.google.flogger:flogger:0.3.1",
-    sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
-)
-
-maven_jar(
-    name = "com_google_common_flogger_system_backend",
-    artifact = "com.google.flogger:flogger-system-backend:0.3.1",
-    sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
-)
-
-maven_jar(
-    name = "com_google_code_findbugs",
-    artifact = "com.google.code.findbugs:jsr305:3.0.2",
-    sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
-)
-
 # You may run setup_android.sh to install Android SDK and NDK.
 android_ndk_repository(
     name = "androidndk",
@@ -293,9 +211,15 @@ android_sdk_repository(
 
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
+    sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42",
-    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz",
-    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
+    patches = [
+        # Bypass checking ios unit test runner when building MP ios applications.
+        "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
 )
 
 load(
@@ -329,3 +253,49 @@ http_archive(
     build_file = "@//third_party:google_toolbox_for_mac.BUILD",
 )
 
+# Maven dependencies.
+
+RULES_JVM_EXTERNAL_TAG = "3.2"
+RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+
+http_archive(
+    name = "rules_jvm_external",
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+# Important: there can only be one maven_install rule. Add new maven deps here.
+maven_install(
+    name = "maven",
+    artifacts = [
+        "junit:junit:4.12",
+        "androidx.test.espresso:espresso-core:3.1.1",
+        "org.hamcrest:hamcrest-library:1.3",
+        "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
+        "androidx.lifecycle:lifecycle-common:2.2.0",
+        "androidx.annotation:annotation:aar:1.1.0",
+        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
+        "androidx.camera:camera-core:aar:1.0.0-alpha06",
+        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
+        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
+        "androidx.core:core:aar:1.1.0-rc03",
+        "androidx.legacy:legacy-support-v4:aar:1.0.0",
+        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
+        "com.google.android.material:material:aar:1.0.0-rc01",
+        "com.google.code.findbugs:jsr305:3.0.2",
+        "com.google.flogger:flogger-system-backend:0.3.1",
+        "com.google.flogger:flogger:0.3.1",
+        "com.google.guava:guava:27.0.1-android",
+    ],
+    repositories = [
+        "https://jcenter.bintray.com",
+        "https://maven.google.com",
+        "https://dl.google.com/dl/android/maven2",
+        "https://repo1.maven.org/maven2",
+    ],
+    fetch_sources = True,
+    version_conflict_policy = "pinned",
+)

mediapipe/BUILD

@@ -14,6 +14,9 @@
 
 licenses(["notice"])  # Apache 2.0
 
+# Note: yes, these need to use "//external:android/crosstool", not
+# @androidndk//:default_crosstool.
+
 config_setting(
     name = "android",
     values = {"crosstool_top": "//external:android/crosstool"},

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -184,23 +184,14 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
   use_local_timestamp_ = spectrogram_options.use_local_timestamp();
 
   if (spectrogram_options.frame_duration_seconds() <= 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Invalid or missing frame_duration_seconds.\n"
-           "frame_duration_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
   }
   if (spectrogram_options.frame_overlap_seconds() >=
       spectrogram_options.frame_duration_seconds()) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Invalid frame_overlap_seconds.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds()
-        << "\nframe_duration_seconds: "
-        << spectrogram_options.frame_duration_seconds();
   }
   if (spectrogram_options.frame_overlap_seconds() < 0.0) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Frame_overlap_seconds is < 0.0.\nframe_overlap_seconds: "
-        << spectrogram_options.frame_overlap_seconds();
   }
 
   TimeSeriesHeader input_header;
@@ -212,9 +203,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   if (!spectrogram_options.allow_multichannel_input() &&
       num_input_channels_ != 1) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "The current setting only supports single-channel input. Please set "
-           "allow_multichannel_input.\n";
   }
 
   frame_duration_samples_ =
@@ -293,10 +282,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   const Matrix& input_stream = cc->Inputs().Index(0).Get<Matrix>();
   if (input_stream.rows() != num_input_channels_) {
-    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+    // TODO: return an error.
-        << "Number of input channels do not correspond to the number of rows "
-        << "in the input matrix: " << num_input_channels_ << "channels vs "
-        << input_stream.rows() << " rows";
   }
 
   cumulative_input_samples_ += input_stream.cols();

mediapipe/calculators/core/BUILD

@@ -230,6 +230,7 @@ cc_library(
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -257,6 +258,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:render_data_cc_proto",
+        "@org_tensorflow//tensorflow/lite:framework",
     ],
     alwayslink = 1,
 )
@@ -779,6 +781,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",
@@ -812,6 +815,38 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "split_normalized_landmark_list_calculator",
+    srcs = ["split_normalized_landmark_list_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "split_normalized_landmark_list_calculator_test",
+    srcs = ["split_normalized_landmark_list_calculator_test.cc"],
+    deps = [
+        ":split_normalized_landmark_list_calculator",
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_library(
     name = "dequantize_byte_array_calculator",
     srcs = ["dequantize_byte_array_calculator.cc"],

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 
 namespace mediapipe {
@@ -37,4 +38,8 @@ typedef BeginLoopCalculator<std::vector<::mediapipe::Detection>>
     BeginLoopDetectionCalculator;
 REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 
+// A calculator to process std::vector<Matrix>.
+typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -51,8 +51,8 @@ namespace mediapipe {
 class ConstantSidePacketCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    const auto& options = cc->Options().GetExtension(
+    const auto& options =
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag),
                  options.packet_size())
         << "Number of output side packets has to be same as number of packets "
@@ -80,8 +80,8 @@ class ConstantSidePacketCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    const auto& options = cc->Options().GetExtension(
+    const auto& options =
-        ::mediapipe::ConstantSidePacketCalculatorOptions::ext);
+        cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>();
     int index = 0;
     for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag);
          id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) {

mediapipe/calculators/core/end_loop_calculator.cc

@@ -20,6 +20,7 @@
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
+#include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
 
@@ -42,4 +43,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::ClassificationList>>
     EndLoopClassificationListCalculator;
 REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
 
+typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
+REGISTER_CALCULATOR(EndLoopTensorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc

@@ -0,0 +1,165 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_  // NOLINT
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+
+// Splits an input packet with NormalizedLandmarkList into
+// multiple NormalizedLandmarkList output packets using the [begin, end) ranges
+// specified in SplitVectorCalculatorOptions. If the option "element_only" is
+// set to true, all ranges should be of size 1 and all outputs will be elements
+// of type NormalizedLandmark. If "element_only" is false, ranges can be
+// non-zero in size and all outputs will be of type NormalizedLandmarkList.
+// If the option "combine_outputs" is set to true, only one output stream can be
+// specified and all ranges of elements will be combined into one
+// NormalizedLandmarkList.
+class SplitNormalizedLandmarkListCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() != 0);
+
+    cc->Inputs().Index(0).Set<NormalizedLandmarkList>();
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    if (options.combine_outputs()) {
+      RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+      cc->Outputs().Index(0).Set<NormalizedLandmarkList>();
+      for (int i = 0; i < options.ranges_size() - 1; ++i) {
+        for (int j = i + 1; j < options.ranges_size(); ++j) {
+          const auto& range_0 = options.ranges(i);
+          const auto& range_1 = options.ranges(j);
+          if ((range_0.begin() >= range_1.begin() &&
+               range_0.begin() < range_1.end()) ||
+              (range_1.begin() >= range_0.begin() &&
+               range_1.begin() < range_0.end())) {
+            return ::mediapipe::InvalidArgumentError(
+                "Ranges must be non-overlapping when using combine_outputs "
+                "option.");
+          }
+        }
+      }
+    } else {
+      if (cc->Outputs().NumEntries() != options.ranges_size()) {
+        return ::mediapipe::InvalidArgumentError(
+            "The number of output streams should match the number of ranges "
+            "specified in the CalculatorOptions.");
+      }
+
+      // Set the output types for each output stream.
+      for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+        if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+            options.ranges(i).begin() >= options.ranges(i).end()) {
+          return ::mediapipe::InvalidArgumentError(
+              "Indices should be non-negative and begin index should be less "
+              "than the end index.");
+        }
+        if (options.element_only()) {
+          if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+            return ::mediapipe::InvalidArgumentError(
+                "Since element_only is true, all ranges should be of size 1.");
+          }
+          cc->Outputs().Index(i).Set<NormalizedLandmark>();
+        } else {
+          cc->Outputs().Index(i).Set<NormalizedLandmarkList>();
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    element_only_ = options.element_only();
+    combine_outputs_ = options.combine_outputs();
+
+    for (const auto& range : options.ranges()) {
+      ranges_.push_back({range.begin(), range.end()});
+      max_range_end_ = std::max(max_range_end_, range.end());
+      total_elements_ += range.end() - range.begin();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const NormalizedLandmarkList& input =
+        cc->Inputs().Index(0).Get<NormalizedLandmarkList>();
+    RET_CHECK_GE(input.landmark_size(), max_range_end_);
+
+    if (combine_outputs_) {
+      NormalizedLandmarkList output;
+      for (int i = 0; i < ranges_.size(); ++i) {
+        for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+          const NormalizedLandmark& input_landmark = input.landmark(j);
+          *output.add_landmark() = input_landmark;
+        }
+      }
+      RET_CHECK_EQ(output.landmark_size(), total_elements_);
+      cc->Outputs().Index(0).AddPacket(
+          MakePacket<NormalizedLandmarkList>(output).At(cc->InputTimestamp()));
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmark>(input.landmark(ranges_[i].first))
+                  .At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          NormalizedLandmarkList output;
+          for (int j = ranges_[i].first; j < ranges_[i].second; ++j) {
+            const NormalizedLandmark& input_landmark = input.landmark(j);
+            *output.add_landmark() = input_landmark;
+          }
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<NormalizedLandmarkList>(output).At(
+                  cc->InputTimestamp()));
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::vector<std::pair<int32, int32>> ranges_;
+  int32 max_range_end_ = -1;
+  int32 total_elements_ = 0;
+  bool element_only_ = false;
+  bool combine_outputs_ = false;
+};
+
+REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator);
+
+}  // namespace mediapipe
+
+// NOLINTNEXTLINE
+#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_

mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc

@@ -0,0 +1,404 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+constexpr float kLocationVal = 3;
+
+class SplitNormalizedLandmarkListCalculatorTest : public ::testing::Test {
+ protected:
+  void TearDown() { expected_landmarks_.reset(); }
+
+  void PrepareNormalizedLandmarkList(int list_size) {
+    // Prepare input landmark list.
+    input_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    expected_landmarks_ = absl::make_unique<NormalizedLandmarkList>();
+    for (int i = 0; i < list_size; ++i) {
+      NormalizedLandmark* landmark = input_landmarks_->add_landmark();
+      landmark->set_x(i * kLocationVal);
+      landmark->set_y(i * kLocationVal);
+      landmark->set_z(i * kLocationVal);
+      // Save the landmarks for comparison after the graph runs.
+      *expected_landmarks_->add_landmark() = *landmark;
+    }
+  }
+
+  void ValidateListOutput(std::vector<Packet>& output_packets,
+                          int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const NormalizedLandmark& expected_landmark =
+          expected_landmarks_->landmark(input_begin_index + i);
+      const NormalizedLandmark& result = output_landmarks.landmark(i);
+      EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+      EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+      EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+    }
+  }
+
+  void ValidateCombinedListOutput(std::vector<Packet>& output_packets,
+                                  int expected_elements,
+                                  std::vector<int>& input_begin_indices,
+                                  std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const NormalizedLandmarkList& output_landmarks =
+        output_packets[0].Get<NormalizedLandmarkList>();
+    ASSERT_EQ(expected_elements, output_landmarks.landmark_size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const NormalizedLandmark& expected_landmark =
+            expected_landmarks_->landmark(i);
+        const NormalizedLandmark& result =
+            output_landmarks.landmark(element_id);
+        EXPECT_FLOAT_EQ(expected_landmark.x(), result.x());
+        EXPECT_FLOAT_EQ(expected_landmark.y(), result.y());
+        EXPECT_FLOAT_EQ(expected_landmark.z(), result.z());
+        element_id++;
+      }
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+
+    const NormalizedLandmark& output_landmark =
+        output_packets[0].Get<NormalizedLandmark>();
+    ASSERT_TRUE(output_landmark.IsInitialized());
+
+    const NormalizedLandmark& expected_landmark =
+        expected_landmarks_->landmark(input_begin_index);
+
+    EXPECT_FLOAT_EQ(expected_landmark.x(), output_landmark.x());
+    EXPECT_FLOAT_EQ(expected_landmark.y(), output_landmark.y());
+    EXPECT_FLOAT_EQ(expected_landmark.z(), output_landmark.z());
+  }
+
+  std::unique_ptr<NormalizedLandmarkList> input_landmarks_ = nullptr;
+  std::unique_ptr<NormalizedLandmarkList> expected_landmarks_ = nullptr;
+  std::unique_ptr<CalculatorRunner> runner_ = nullptr;
+};
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateListOutput(range_0_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/0);
+  ValidateListOutput(range_1_packets, /*expected_elements=*/3,
+                     /*input_begin_index=*/1);
+  ValidateListOutput(range_2_packets, /*expected_elements=*/1,
+                     /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 0 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because of an invalid range (begin == end).
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOutputStreamCountTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidCombineOutputsMultipleOutputsTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       InvalidOverlappingRangesTest) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateElementOutput(range_0_packets,
+                        /*input_begin_index=*/0);
+  ValidateElementOutput(range_1_packets,
+                        /*input_begin_index=*/2);
+  ValidateElementOutput(range_2_packets,
+                        /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) {
+  PrepareNormalizedLandmarkList(/*list_size=*/5);
+  ASSERT_NE(input_landmarks_, nullptr);
+
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedListOutput(range_0_packets, /*expected_elements=*/3,
+                             input_begin_indices, input_end_indices);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("landmarks_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitNormalizedLandmarkListCalculatorTest,
+       ElementOnlyDisablesVectorOutputs) {
+  // Prepare a graph to use the SplitNormalizedLandmarkListCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "landmarks_in"
+            node {
+              calculator: "SplitNormalizedLandmarkListCalculator"
+              input_stream: "landmarks_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
@@ -57,6 +58,9 @@ typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
     SplitNormalizedRectVectorCalculator;
 REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
 
+typedef SplitVectorCalculator<Matrix, false> SplitMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitMatrixVectorCalculator);
+
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
     MovableSplitGlBufferVectorCalculator;

mediapipe/calculators/image/BUILD

@@ -86,6 +86,14 @@ mediapipe_cc_proto_library(
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "mask_overlay_calculator_cc_proto",
     srcs = ["mask_overlay_calculator.proto"],
@@ -172,6 +180,7 @@ cc_library(
     srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":opencv_encoded_image_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/port:opencv_imgcodecs",
@@ -557,6 +566,27 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "feature_detector_calculator_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "feature_detector_calculator_cc_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":feature_detector_calculator_proto"],
+)
+
 cc_library(
     name = "mask_overlay_calculator",
     srcs = ["mask_overlay_calculator.cc"],
@@ -572,3 +602,30 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "feature_detector_calculator",
+    srcs = ["feature_detector_calculator.cc"],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [
+        ":feature_detector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:threadpool",
+        "//mediapipe/framework/tool:options_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/image/feature_detector_calculator.cc

@@ -0,0 +1,210 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/synchronization/blocking_counter.h"
+#include "mediapipe/calculators/image/feature_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/threadpool.h"
+#include "mediapipe/framework/tool/options_util.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+const char kOptionsTag[] = "OPTIONS";
+const int kPatchSize = 32;
+const int kNumThreads = 16;
+
+// A calculator to apply local feature detection.
+// Input stream:
+//   IMAGE: Input image frame of type ImageFrame from video stream.
+// Output streams:
+//   FEATURES: The detected keypoints from input image as vector<cv::KeyPoint>.
+//   PATCHES:  Optional output the extracted patches as vector<cv::Mat>
+class FeatureDetectorCalculator : public CalculatorBase {
+ public:
+  ~FeatureDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  FeatureDetectorCalculatorOptions options_;
+  cv::Ptr<cv::Feature2D> feature_detector_;
+  std::unique_ptr<::mediapipe::ThreadPool> pool_;
+
+  // Create image pyramid based on input image.
+  void ComputeImagePyramid(const cv::Mat& input_image,
+                           std::vector<cv::Mat>* image_pyramid);
+
+  // Extract the patch for single feature with image pyramid.
+  cv::Mat ExtractPatch(const cv::KeyPoint& feature,
+                       const std::vector<cv::Mat>& image_pyramid);
+};
+
+REGISTER_CALCULATOR(FeatureDetectorCalculator);
+
+::mediapipe::Status FeatureDetectorCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("IMAGE")) {
+    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag("FEATURES")) {
+    cc->Outputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    cc->Outputs().Tag("LANDMARKS").Set<NormalizedLandmarkList>();
+  }
+  if (cc->Outputs().HasTag("PATCHES")) {
+    cc->Outputs().Tag("PATCHES").Set<std::vector<TfLiteTensor>>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ =
+      tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag)
+          .GetExtension(FeatureDetectorCalculatorOptions::ext);
+  feature_detector_ = cv::ORB::create(
+      options_.max_features(), options_.scale_factor(),
+      options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE);
+  pool_ = absl::make_unique<::mediapipe::ThreadPool>("ThreadPool", kNumThreads);
+  pool_->StartWorkers();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp& timestamp = cc->InputTimestamp();
+  if (timestamp == Timestamp::PreStream()) {
+    // Indicator packet.
+    return ::mediapipe::OkStatus();
+  }
+  InputStream* input_frame = &(cc->Inputs().Tag("IMAGE"));
+  cv::Mat input_view = formats::MatView(&input_frame->Get<ImageFrame>());
+  cv::Mat grayscale_view;
+  cv::cvtColor(input_view, grayscale_view, cv::COLOR_RGB2GRAY);
+
+  std::vector<cv::KeyPoint> keypoints;
+  feature_detector_->detect(grayscale_view, keypoints);
+  if (keypoints.size() > options_.max_features()) {
+    keypoints.resize(options_.max_features());
+  }
+
+  if (cc->Outputs().HasTag("FEATURES")) {
+    auto features_ptr = absl::make_unique<std::vector<cv::KeyPoint>>(keypoints);
+    cc->Outputs().Tag("FEATURES").Add(features_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    auto landmarks_ptr = absl::make_unique<NormalizedLandmarkList>();
+    for (int j = 0; j < keypoints.size(); ++j) {
+      auto feature_landmark = landmarks_ptr->add_landmark();
+      feature_landmark->set_x(keypoints[j].pt.x / grayscale_view.cols);
+      feature_landmark->set_y(keypoints[j].pt.y / grayscale_view.rows);
+    }
+    cc->Outputs().Tag("LANDMARKS").Add(landmarks_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("PATCHES")) {
+    std::vector<cv::Mat> image_pyramid;
+    ComputeImagePyramid(grayscale_view, &image_pyramid);
+    std::vector<cv::Mat> patch_mat;
+    patch_mat.resize(keypoints.size());
+    absl::BlockingCounter counter(keypoints.size());
+    for (int i = 0; i < keypoints.size(); i++) {
+      pool_->Schedule(
+          [this, &image_pyramid, &keypoints, &patch_mat, i, &counter] {
+            patch_mat[i] = ExtractPatch(keypoints[i], image_pyramid);
+            counter.DecrementCount();
+          });
+    }
+    counter.Wait();
+    const int batch_size = options_.max_features();
+    auto patches = absl::make_unique<std::vector<TfLiteTensor>>();
+    TfLiteTensor tensor;
+    tensor.type = kTfLiteFloat32;
+    tensor.dims = TfLiteIntArrayCreate(4);
+    tensor.dims->data[0] = batch_size;
+    tensor.dims->data[1] = kPatchSize;
+    tensor.dims->data[2] = kPatchSize;
+    tensor.dims->data[3] = 1;
+    int num_bytes = batch_size * kPatchSize * kPatchSize * sizeof(float);
+    tensor.data.data = malloc(num_bytes);
+    tensor.bytes = num_bytes;
+    tensor.allocation_type = kTfLiteArenaRw;
+    float* tensor_buffer = tensor.data.f;
+    for (int i = 0; i < keypoints.size(); i++) {
+      for (int j = 0; j < patch_mat[i].rows; ++j) {
+        for (int k = 0; k < patch_mat[i].cols; ++k) {
+          *tensor_buffer++ = patch_mat[i].at<uchar>(j, k) / 128.0f - 1.0f;
+        }
+      }
+    }
+    for (int i = keypoints.size() * kPatchSize * kPatchSize; i < num_bytes / 4;
+         i++) {
+      *tensor_buffer++ = 0;
+    }
+
+    patches->emplace_back(tensor);
+    cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FeatureDetectorCalculator::ComputeImagePyramid(
+    const cv::Mat& input_image, std::vector<cv::Mat>* image_pyramid) {
+  cv::Mat tmp_image = input_image;
+  cv::Mat src_image = input_image;
+  for (int i = 0; i < options_.pyramid_level(); ++i) {
+    image_pyramid->push_back(src_image);
+    cv::resize(src_image, tmp_image, cv::Size(), 1.0f / options_.scale_factor(),
+               1.0f / options_.scale_factor());
+    src_image = tmp_image;
+  }
+}
+
+cv::Mat FeatureDetectorCalculator::ExtractPatch(
+    const cv::KeyPoint& feature, const std::vector<cv::Mat>& image_pyramid) {
+  cv::Mat img = image_pyramid[feature.octave];
+  float scale_factor = 1 / pow(options_.scale_factor(), feature.octave);
+  cv::Point2f center =
+      cv::Point2f(feature.pt.x * scale_factor, feature.pt.y * scale_factor);
+  cv::Mat rot = cv::getRotationMatrix2D(center, feature.angle, 1.0);
+  rot.at<double>(0, 2) += kPatchSize / 2 - center.x;
+  rot.at<double>(1, 2) += kPatchSize / 2 - center.y;
+  cv::Mat cropped_img;
+  // perform the affine transformation
+  cv::warpAffine(img, cropped_img, rot, cv::Size(kPatchSize, kPatchSize),
+                 cv::INTER_LINEAR);
+  return cropped_img;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/feature_detector_calculator.proto

@@ -0,0 +1,24 @@
+// Options for FeatureDetectorCalculator
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FeatureDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional FeatureDetectorCalculatorOptions ext = 278741680;
+  }
+
+  // Set to true if output patches, otherwise only output cv::KeyPoint
+  optional bool output_patch = 1;
+
+  // The max number of detected features.
+  optional int32 max_features = 2 [default = 200];
+
+  // The number of pyramid levels.
+  optional int32 pyramid_level = 3 [default = 4];
+
+  // Pyramid decimation ratio.
+  optional float scale_factor = 4 [default = 1.2];
+}

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -219,8 +219,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
-  auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
+  RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height());
-      GetCropSpecs(cc, input_img.Width(), input_img.Height());
+  int target_width = specs.width, target_height = specs.height,
+      rect_center_x = specs.center_x, rect_center_y = specs.center_y;
+  float rotation = specs.rotation;
 
   // Get border mode and value for OpenCV.
   int border_mode;
@@ -403,8 +405,10 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
                                                   int src_width, int src_height,
                                                   int* dst_width,
                                                   int* dst_height) {
-  auto [crop_width, crop_height, x_center, y_center, rotation] =
+  RectSpec specs = GetCropSpecs(cc, src_width, src_height);
-      GetCropSpecs(cc, src_width, src_height);
+  int crop_width = specs.width, crop_height = specs.height,
+      x_center = specs.center_x, y_center = specs.center_y;
+  float rotation = specs.rotation;
 
   const float half_width = crop_width / 2.0f;
   const float half_height = crop_height / 2.0f;

mediapipe/calculators/image/image_properties_calculator.cc

@@ -19,6 +19,11 @@
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
+namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+}  // namespace
+
 namespace mediapipe {
 
 // Extracts image properties from the input image and outputs the properties.
@@ -40,13 +45,14 @@ namespace mediapipe {
 class ImagePropertiesCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-    if (cc->Inputs().HasTag("IMAGE")) {
+              cc->Inputs().HasTag(kGpuBufferTag));
-      cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    if (cc->Inputs().HasTag(kImageFrameTag)) {
+      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU")) {
+    if (cc->Inputs().HasTag(kGpuBufferTag)) {
-      cc->Inputs().Tag("IMAGE_GPU").Set<::mediapipe::GpuBuffer>();
+      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
     }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
@@ -66,16 +72,17 @@ class ImagePropertiesCalculator : public CalculatorBase {
     int width;
     int height;
 
-    if (cc->Inputs().HasTag("IMAGE") && !cc->Inputs().Tag("IMAGE").IsEmpty()) {
+    if (cc->Inputs().HasTag(kImageFrameTag) &&
-      const auto& image = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
+      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
       width = image.Width();
       height = image.Height();
     }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().HasTag("IMAGE_GPU") &&
+    if (cc->Inputs().HasTag(kGpuBufferTag) &&
-        !cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
       const auto& image =
-          cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
       width = image.width();
       height = image.height();
     }

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -47,6 +47,9 @@ namespace mediapipe {
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+
 int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
   switch (rotation) {
     case mediapipe::RotationMode_Mode_UNKNOWN:
@@ -95,7 +98,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Scales, rotates, and flips images horizontally or vertically.
 //
 // Input:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE: ImageFrame representing the input image.
 //   IMAGE_GPU: GpuBuffer representing the input image.
 //
@@ -113,7 +116,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   corresponding field in the calculator options.
 //
 // Output:
-//   One of the following two tags:
+//   One of the following tags:
 //   IMAGE - ImageFrame representing the output image.
 //   IMAGE_GPU - GpuBuffer representing the output image.
 //
@@ -152,7 +155,8 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 // Note: To enable horizontal or vertical flipping, specify them in the
 // calculator options. Flipping is applied after rotation.
 //
-// Note: Only scale mode STRETCH is currently supported on CPU.
+// Note: Input defines output, so only matchig types supported:
+// IMAGE -> IMAGE  or  IMAGE_GPU -> IMAGE_GPU
 //
 class ImageTransformationCalculator : public CalculatorBase {
  public:
@@ -186,7 +190,7 @@ class ImageTransformationCalculator : public CalculatorBase {
 
   bool use_gpu_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  GlCalculatorHelper helper_;
+  GlCalculatorHelper gpu_helper_;
   std::unique_ptr<QuadRenderer> rgb_renderer_;
   std::unique_ptr<QuadRenderer> yuv_renderer_;
   std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
@@ -197,21 +201,22 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 // static
 ::mediapipe::Status ImageTransformationCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  // Only one input can be set, and the output type must match.
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE"));
+    RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
+    RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
-    cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
-    cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+    cc->Outputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -259,7 +264,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   options_ = cc->Options<ImageTransformationCalculatorOptions>();
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
   }
 
@@ -300,7 +305,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     // Let the helper access the GL context information.
-    MP_RETURN_IF_ERROR(helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -328,18 +333,14 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-    if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+    if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
       return ::mediapipe::OkStatus();
     }
-    return helper_.RunInGlContext(
+    return gpu_helper_.RunInGlContext(
         [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
+    if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
-      // Image is missing, hence no way to produce output image. (Timestamp
-      // bound will be updated automatically.)
       return ::mediapipe::OkStatus();
     }
     return RenderCpu(cc);
@@ -354,7 +355,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     QuadRenderer* rgb_renderer = rgb_renderer_.release();
     QuadRenderer* yuv_renderer = yuv_renderer_.release();
     QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release();
-    helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
+    gpu_helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
       if (rgb_renderer) {
         rgb_renderer->GlTeardown();
         delete rgb_renderer;
@@ -376,18 +377,21 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
 ::mediapipe::Status ImageTransformationCalculator::RenderCpu(
     CalculatorContext* cc) {
-  int input_width = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Width();
+  cv::Mat input_mat;
-  int input_height = cc->Inputs().Tag("IMAGE").Get<ImageFrame>().Height();
+  mediapipe::ImageFormat::Format format;
 
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  const auto& input = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-  cv::Mat input_mat = formats::MatView(&input_img);
+  input_mat = formats::MatView(&input);
-  cv::Mat scaled_mat;
+  format = input.Format();
 
+  const int input_width = input_mat.cols;
+  const int input_height = input_mat.rows;
   if (!output_height_ || !output_width_) {
     output_height_ = input_height;
     output_width_ = input_width;
   }
 
+  cv::Mat scaled_mat;
   if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
     cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
   } else {
@@ -444,10 +448,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   }
 
   std::unique_ptr<ImageFrame> output_frame(
-      new ImageFrame(input_img.Format(), output_width, output_height));
+      new ImageFrame(format, output_width, output_height));
   cv::Mat output_mat = formats::MatView(output_frame.get());
   flipped_mat.copyTo(output_mat);
-  cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_frame.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -455,8 +461,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  int input_width = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().width();
+  const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<GpuBuffer>();
-  int input_height = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().height();
+  const int input_width = input.width();
+  const int input_height = input.height();
 
   int output_width;
   int output_height;
@@ -472,7 +479,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         .Add(padding.release(), cc->InputTimestamp());
   }
 
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>();
   QuadRenderer* renderer = nullptr;
   GlTexture src1;
 
@@ -486,11 +492,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
                                  {"video_frame_y", "video_frame_uv"}));
     }
     renderer = yuv_renderer_.get();
-    src1 = helper_.CreateSourceTexture(input, 0);
+    src1 = gpu_helper_.CreateSourceTexture(input, 0);
   } else  // NOLINT(readability/braces)
 #endif    // iOS
   {
-    src1 = helper_.CreateSourceTexture(input);
+    src1 = gpu_helper_.CreateSourceTexture(input);
 #if defined(TEXTURE_EXTERNAL_OES)
     if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
       if (!ext_rgb_renderer_) {
@@ -516,10 +522,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   mediapipe::FrameRotation rotation =
       mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
 
-  auto dst = helper_.CreateDestinationTexture(output_width, output_height,
+  auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
-                                              input.format());
+                                                  input.format());
 
-  helper_.BindFramebuffer(dst);  // GL_TEXTURE0
+  gpu_helper_.BindFramebuffer(dst);  // GL_TEXTURE0
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 
@@ -534,8 +540,8 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   // Execute GL commands, before getting result.
   glFlush();
 
-  auto output = dst.GetFrame<GpuBuffer>();
+  auto output = dst.template GetFrame<GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
@@ -34,7 +35,11 @@ namespace mediapipe {
 class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_;
 };
 
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract(
@@ -44,13 +49,29 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open(
+    CalculatorContext* cc) {
+  options_ =
+      cc->Options<mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions>();
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process(
     CalculatorContext* cc) {
   const std::string& contents = cc->Inputs().Index(0).Get<std::string>();
   const std::vector<char> contents_vector(contents.begin(), contents.end());
-  cv::Mat decoded_mat =
+  cv::Mat decoded_mat;
-      cv::imdecode(contents_vector, -1 /* return the loaded image as-is */);
+  if (options_.apply_orientation_from_exif_data()) {
-
+    // We want to respect the orientation from the EXIF data, which
+    // IMREAD_UNCHANGED ignores, but otherwise we want to be as permissive as
+    // possible with our reading flags. Therefore, we use IMREAD_ANYCOLOR and
+    // IMREAD_ANYDEPTH.
+    decoded_mat = cv::imdecode(contents_vector,
+                               cv::IMREAD_ANYCOLOR | cv::IMREAD_ANYDEPTH);
+  } else {
+    // Return the loaded image as-is
+    decoded_mat = cv::imdecode(contents_vector, cv::IMREAD_UNCHANGED);
+  }
   ImageFormat::Format image_format = ImageFormat::UNKNOWN;
   cv::Mat output_mat;
   switch (decoded_mat.channels()) {
@@ -70,7 +91,8 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
              << "Unsupported number of channels: " << decoded_mat.channels();
   }
   std::unique_ptr<ImageFrame> output_frame = absl::make_unique<ImageFrame>(
-      image_format, decoded_mat.size().width, decoded_mat.size().height);
+      image_format, decoded_mat.size().width, decoded_mat.size().height,
+      ImageFrame::kGlDefaultAlignmentBoundary);
   output_mat.copyTo(formats::MatView(output_frame.get()));
   cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
   return ::mediapipe::OkStatus();

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message OpenCvEncodedImageToImageFrameCalculatorOptions {
+  extend CalculatorOptions {
+    optional OpenCvEncodedImageToImageFrameCalculatorOptions ext = 303447308;
+  }
+
+  // If set, we will attempt to automatically apply the orientation specified by
+  // the image's EXIF data when loading the image. Otherwise, the image data
+  // will be loaded as-is.
+  optional bool apply_orientation_from_exif_data = 1 [default = false];
+}

mediapipe/calculators/image/recolor_calculator.cc

@@ -32,6 +32,11 @@
 
 namespace {
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kMaskCpuTag[] = "MASK";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -112,39 +117,41 @@ REGISTER_CALCULATOR(RecolorCalculator);
   bool use_gpu = false;
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Inputs().HasTag("MASK_GPU")) {
+  if (cc->Inputs().HasTag(kMaskGpuTag)) {
-    cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Inputs().HasTag("MASK")) {
+  if (cc->Inputs().HasTag(kMaskCpuTag)) {
-    cc->Inputs().Tag("MASK").Set<ImageFrame>();
+    cc->Inputs().Tag(kMaskCpuTag).Set<ImageFrame>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU)
-  if (cc->Outputs().HasTag("IMAGE_GPU")) {
+  if (cc->Outputs().HasTag(kGpuBufferTag)) {
-    cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Outputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
-  if (cc->Outputs().HasTag("IMAGE")) {
+  if (cc->Outputs().HasTag(kImageFrameTag)) {
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+    cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
   }
 
   // Confirm only one of the input streams is present.
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
+            cc->Inputs().HasTag(kGpuBufferTag));
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) ^
+            cc->Outputs().HasTag(kGpuBufferTag));
 
   if (use_gpu) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -158,7 +165,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 ::mediapipe::Status RecolorCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -201,12 +208,12 @@ REGISTER_CALCULATOR(RecolorCalculator);
 }
 
 ::mediapipe::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
   // Get inputs and setup output.
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  const auto& input_img = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-  const auto& mask_img = cc->Inputs().Tag("MASK").Get<ImageFrame>();
+  const auto& mask_img = cc->Inputs().Tag(kMaskCpuTag).Get<ImageFrame>();
 
   cv::Mat input_mat = formats::MatView(&input_img);
   cv::Mat mask_mat = formats::MatView(&mask_img);
@@ -254,19 +261,21 @@ REGISTER_CALCULATOR(RecolorCalculator);
     }
   }
 
-  cc->Outputs().Tag("IMAGE").Add(output_img.release(), cc->InputTimestamp());
+  cc->Outputs()
+      .Tag(kImageFrameTag)
+      .Add(output_img.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   // Get inputs and setup output.
-  const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
+  const Packet& input_packet = cc->Inputs().Tag(kGpuBufferTag).Value();
-  const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value();
+  const Packet& mask_packet = cc->Inputs().Tag(kMaskGpuTag).Value();
 
   const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
   const auto& mask_buffer = mask_packet.Get<mediapipe::GpuBuffer>();
@@ -296,7 +305,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
 
   // Send result image in GPU packet.
   auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
 
   // Cleanup
   img_tex.Release();

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -244,7 +244,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
   ::mediapipe::Status VerifySequence() {
     std::string error_msg = "Missing features - ";
     bool all_present = true;
-    for (auto iter : features_present_) {
+    for (const auto& iter : features_present_) {
       if (!iter.second) {
         all_present = false;
         absl::StrAppend(&error_msg, iter.first, ", ");

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -51,10 +51,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
+// If options.convert_signature_to_tags() is set, will convert letters to
-// and replace /'s with _'s. If set, this enables the standard SavedModel
+// uppercase and replace /'s and -'s with _'s. This enables the standard
-// classification, regression, and prediction signatures to be used as
+// SavedModel classification, regression, and prediction signatures to be used
-// uppercase INPUTS and OUTPUTS tags for streams.
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelCalculatorOptions& options) {
@@ -64,6 +65,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;
@@ -124,7 +126,7 @@ class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelCalculatorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
+  // Whether to convert the signature keys to uppercase as well as switch /'s
-  // _'s, which enables standard signatures to be used as Tags.
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -53,10 +53,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
+// If options.convert_signature_to_tags() is set, will convert letters to
-// and replace /'s with _'s. If set, this enables the standard SavedModel
+// uppercase and replace /'s and -'s with _'s. This enables the standard
-// classification, regression, and prediction signatures to be used as
+// SavedModel classification, regression, and prediction signatures to be used
-// uppercase INPUTS and OUTPUTS tags for streams.
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelGeneratorOptions& options) {
@@ -66,6 +67,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;
@@ -119,7 +121,7 @@ class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator {
     // Set user specified tags properly.
     // If no tags specified will use tensorflow::kSavedModelTagServe by default.
     std::unordered_set<std::string> tags_set;
-    for (std::string tag : options.saved_model_tag()) {
+    for (const std::string& tag : options.saved_model_tag()) {
       tags_set.insert(tag);
     }
     if (tags_set.empty()) {

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelGeneratorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
+  // Whether to convert the signature keys to uppercase as well as switch /'s
-  // _'s, which enables standard signatures to be used as Tags.
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tflite/BUILD

@@ -243,6 +243,7 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
         ],
         "//conditions:default": [
+            "//mediapipe/util/tflite:tflite_gpu_runner",
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gpu_buffer",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
@@ -451,7 +452,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -63,6 +63,10 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
 typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
     ColMajorMatrixXf;
 
+constexpr char kImageFrameTag[] = "IMAGE";
+constexpr char kGpuBufferTag[] = "IMAGE_GPU";
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -124,6 +128,9 @@ struct GPUData {
 //  GPU tensors are currently only supported on mobile platforms.
 //  This calculator uses FixedSizeInputStreamHandler by default.
 //
+// Note: Input defines output, so only these type sets are supported:
+// IMAGE -> TENSORS | IMAGE_GPU -> TENSORS_GPU | MATRIX -> TENSORS
+//
 class TfLiteConverterCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
@@ -138,9 +145,9 @@ class TfLiteConverterCalculator : public CalculatorBase {
   template <class T>
   ::mediapipe::Status NormalizeImage(const ImageFrame& image_frame,
                                      bool zero_center, bool flip_vertically,
-                                     float* tensor_buffer);
+                                     float* tensor_ptr);
   ::mediapipe::Status CopyMatrixToTensor(const Matrix& matrix,
-                                         float* tensor_buffer);
+                                         float* tensor_ptr);
   ::mediapipe::Status ProcessCPU(CalculatorContext* cc);
   ::mediapipe::Status ProcessGPU(CalculatorContext* cc);
 
@@ -166,33 +173,35 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::GetContract(
     CalculatorContract* cc) {
-  const bool has_image_tag = cc->Inputs().HasTag("IMAGE");
-  const bool has_image_gpu_tag = cc->Inputs().HasTag("IMAGE_GPU");
-  const bool has_matrix_tag = cc->Inputs().HasTag("MATRIX");
   // Confirm only one of the input streams is present.
-  RET_CHECK(has_image_tag ^ has_image_gpu_tag ^ has_matrix_tag &&
+  RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-            !(has_image_tag && has_image_gpu_tag && has_matrix_tag));
+            cc->Inputs().HasTag(kGpuBufferTag) ^ cc->Inputs().HasTag("MATRIX"));
 
   // Confirm only one of the output streams is present.
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("IMAGE")) cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
-  if (cc->Inputs().HasTag("MATRIX")) cc->Inputs().Tag("MATRIX").Set<Matrix>();
+    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+  }
+  if (cc->Inputs().HasTag("MATRIX")) {
+    cc->Inputs().Tag("MATRIX").Set<Matrix>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kGpuBufferTag)) {
-    cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
+  if (cc->Outputs().HasTag(kTensorsTag)) {
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
+  }
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -216,8 +225,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   MP_RETURN_IF_ERROR(LoadOptions(cc));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU") ||
+  if (cc->Inputs().HasTag(kGpuBufferTag) ||
-      cc->Outputs().HasTag("IMAGE_OUT_GPU")) {
+      cc->Outputs().HasTag(kGpuBufferTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     use_gpu_ = true;
 #else
@@ -227,8 +236,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   if (use_gpu_) {
     // Cannot mix CPU/GPU streams.
-    RET_CHECK(cc->Inputs().HasTag("IMAGE_GPU") &&
+    RET_CHECK(cc->Inputs().HasTag(kGpuBufferTag) &&
-              cc->Outputs().HasTag("TENSORS_GPU"));
+              cc->Outputs().HasTag(kTensorsGpuTag));
     // Cannot use quantization.
     use_quantized_tensors_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -248,7 +257,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-    // GpuBuffer to tflite::gpu::GlBuffer conversion.
     if (!initialized_) {
       MP_RETURN_IF_ERROR(InitGpu(cc));
       initialized_ = true;
@@ -259,7 +267,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     // Convert to CPU tensors or Matrix type.
     MP_RETURN_IF_ERROR(ProcessCPU(cc));
   }
-
   return ::mediapipe::OkStatus();
 }
 
@@ -275,24 +282,26 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::ProcessCPU(
     CalculatorContext* cc) {
-  if (cc->Inputs().HasTag("IMAGE")) {
+  if (cc->Inputs().HasTag(kImageFrameTag)) {
     // CPU ImageFrame to TfLiteTensor conversion.
 
-    const auto& image_frame = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+    const auto& image_frame =
+        cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
     const int height = image_frame.Height();
     const int width = image_frame.Width();
     const int channels = image_frame.NumberOfChannels();
     const int channels_preserved = std::min(channels, max_num_channels_);
+    const mediapipe::ImageFormat::Format format = image_frame.Format();
 
     if (!initialized_) {
-      if (!(image_frame.Format() == mediapipe::ImageFormat::SRGBA ||
+      if (!(format == mediapipe::ImageFormat::SRGBA ||
-            image_frame.Format() == mediapipe::ImageFormat::SRGB ||
+            format == mediapipe::ImageFormat::SRGB ||
-            image_frame.Format() == mediapipe::ImageFormat::GRAY8 ||
+            format == mediapipe::ImageFormat::GRAY8 ||
-            image_frame.Format() == mediapipe::ImageFormat::VEC32F1))
+            format == mediapipe::ImageFormat::VEC32F1))
         RET_CHECK_FAIL() << "Unsupported CPU input format.";
       TfLiteQuantization quant;
       if (use_quantized_tensors_) {
-        RET_CHECK(image_frame.Format() != mediapipe::ImageFormat::VEC32F1)
+        RET_CHECK(format != mediapipe::ImageFormat::VEC32F1)
             << "Only 8-bit input images are supported for quantization.";
         quant.type = kTfLiteAffineQuantization;
         quant.params = nullptr;
@@ -349,8 +358,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   } else if (cc->Inputs().HasTag("MATRIX")) {
     // CPU Matrix to TfLiteTensor conversion.
 
@@ -371,15 +381,16 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     interpreter_->ResizeInputTensor(tensor_idx, {height, width, channels});
     interpreter_->AllocateTensors();
 
-    float* tensor_buffer = tensor->data.f;
+    float* tensor_ptr = tensor->data.f;
-    RET_CHECK(tensor_buffer);
+    RET_CHECK(tensor_ptr);
 
-    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_buffer));
+    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_ptr));
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -389,7 +400,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // GpuBuffer to tflite::gpu::GlBuffer conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   MP_RETURN_IF_ERROR(
       gpu_helper_.RunInGlContext([this, &input]() -> ::mediapipe::Status {
         // Convert GL texture into TfLite GlBuffer (SSBO).
@@ -421,11 +433,12 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
         return ::mediapipe::OkStatus();
       }));
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
   // GpuBuffer to id<MTLBuffer> conversion.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
 
   id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
@@ -457,7 +470,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
                     commandBuffer:command_buffer];
 
   cc->Outputs()
-      .Tag("TENSORS_GPU")
+      .Tag(kTensorsGpuTag)
       .Add(output_tensors.release(), cc->InputTimestamp());
 #else
   RET_CHECK_FAIL() << "GPU processing is not enabled.";
@@ -469,7 +482,8 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 ::mediapipe::Status TfLiteConverterCalculator::InitGpu(CalculatorContext* cc) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
   // Get input image sizes.
-  const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+  const auto& input =
+      cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
   mediapipe::ImageFormat::Format format =
       mediapipe::ImageFormatForGpuBufferFormat(input.format());
   gpu_data_out_ = absl::make_unique<GPUData>();
@@ -612,7 +626,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   CHECK_LE(max_num_channels_, 4);
   CHECK_NE(max_num_channels_, 2);
 #if defined(MEDIAPIPE_IOS)
-  if (cc->Inputs().HasTag("IMAGE_GPU"))
+  if (cc->Inputs().HasTag(kGpuBufferTag))
     // Currently on iOS, tflite gpu input tensor must be 4 channels,
     // so input image must be 4 channels also (checked in InitGpu).
     max_num_channels_ = 4;
@@ -627,7 +641,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 template <class T>
 ::mediapipe::Status TfLiteConverterCalculator::NormalizeImage(
     const ImageFrame& image_frame, bool zero_center, bool flip_vertically,
-    float* tensor_buffer) {
+    float* tensor_ptr) {
   const int height = image_frame.Height();
   const int width = image_frame.Width();
   const int channels = image_frame.NumberOfChannels();
@@ -651,7 +665,7 @@ template <class T>
         (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
     for (int j = 0; j < width; ++j) {
       for (int c = 0; c < channels_preserved; ++c) {
-        *tensor_buffer++ = *image_ptr++ / div - sub;
+        *tensor_ptr++ = *image_ptr++ / div - sub;
       }
       image_ptr += channels_ignored;
     }
@@ -661,14 +675,14 @@ template <class T>
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::CopyMatrixToTensor(
-    const Matrix& matrix, float* tensor_buffer) {
+    const Matrix& matrix, float* tensor_ptr) {
   if (row_major_matrix_) {
-    auto matrix_map = Eigen::Map<RowMajorMatrixXf>(tensor_buffer, matrix.rows(),
+    auto matrix_map =
-                                                   matrix.cols());
+        Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   } else {
-    auto matrix_map = Eigen::Map<ColMajorMatrixXf>(tensor_buffer, matrix.rows(),
+    auto matrix_map =
-                                                   matrix.cols());
+        Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
     matrix_map = matrix;
   }
 

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -23,9 +23,9 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
 #include "mediapipe/util/cpu_util.h"
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
@@ -36,6 +36,7 @@
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/util/tflite/tflite_gpu_runner.h"
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
@@ -75,6 +76,9 @@ typedef id<MTLBuffer> GpuTensor;
 
 // Round up n to next multiple of m.
 size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 #if defined(MEDIAPIPE_EDGE_TPU)
@@ -121,7 +125,7 @@ struct GPUData {
 
 // Returns number of threads to configure XNNPACK delegate with.
 // (Equal to user provided value if specified.  Otherwise, it returns number of
-// high cores (hard-coded to 1 for __EMSCRIPTEN__))
+// high cores (hard-coded to 1 for Emscripten without Threads extension))
 int GetXnnpackNumThreads(
     const mediapipe::TfLiteInferenceCalculatorOptions& opts) {
   static constexpr int kDefaultNumThreads = -1;
@@ -129,11 +133,11 @@ int GetXnnpackNumThreads(
       opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) {
     return opts.delegate().xnnpack().num_threads();
   }
-#if !defined(__EMSCRIPTEN__)
+#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__)
   return InferHigherCoreIds().size();
 #else
   return 1;
-#endif  // !__EMSCRIPTEN__
+#endif  // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__
 }
 
 // Calculator Header Section
@@ -219,6 +223,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   ::mediapipe::Status LoadModel(CalculatorContext* cc);
   ::mediapipe::StatusOr<Packet> GetModelAsPacket(const CalculatorContext& cc);
   ::mediapipe::Status LoadDelegate(CalculatorContext* cc);
+  ::mediapipe::Status InitTFLiteGPURunner();
 
   Packet model_packet_;
   std::unique_ptr<tflite::Interpreter> interpreter_;
@@ -228,6 +233,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
 #elif defined(MEDIAPIPE_IOS)
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::vector<std::unique_ptr<GPUData>> gpu_data_in_;
@@ -245,6 +251,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   bool gpu_input_ = false;
   bool gpu_output_ = false;
   bool use_quantized_tensors_ = false;
+
+  bool use_advanced_gpu_api_ = false;
 };
 REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
@@ -252,10 +260,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Inputs().HasTag(kTensorsTag) ^
-            cc->Inputs().HasTag("TENSORS_GPU"));
+            cc->Inputs().HasTag(kTensorsGpuTag));
-  RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
+  RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^
-            cc->Outputs().HasTag("TENSORS_GPU"));
+            cc->Outputs().HasTag(kTensorsGpuTag));
 
   const auto& options =
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
@@ -266,26 +274,26 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   bool use_gpu =
       options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS"))
+  if (cc->Inputs().HasTag(kTensorsTag))
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU input is compatible with GPU delegate only.";
 
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
-  if (cc->Outputs().HasTag("TENSORS"))
+  if (cc->Outputs().HasTag(kTensorsTag))
-    cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Outputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
     RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
         << "GPU output is compatible with GPU delegate only.";
 
-    cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Outputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -320,27 +328,31 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
   gpu_inference_ = options.use_gpu();
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_input_ = true;
     gpu_inference_ = true;  // Inference must be on GPU also.
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  if (cc->Outputs().HasTag("TENSORS_GPU")) {
+  if (cc->Outputs().HasTag(kTensorsGpuTag)) {
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_output_ = true;
-    RET_CHECK(cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU output must also have GPU Input.";
 #else
-    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
+    RET_CHECK(!cc->Inputs().HasTag(kTensorsGpuTag))
         << "GPU processing not enabled.";
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
+  const auto& calculator_opts =
+      cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
+  use_advanced_gpu_api_ = false;
+
   MP_RETURN_IF_ERROR(LoadModel(cc));
 
   if (gpu_inference_) {
@@ -352,8 +364,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+    MP_RETURN_IF_ERROR(
-        [this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
+        gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
+          return use_advanced_gpu_api_ ? InitTFLiteGPURunner()
+                                       : LoadDelegate(cc);
+        }));
+    if (use_advanced_gpu_api_) return ::mediapipe::OkStatus();
 #else
     MP_RETURN_IF_ERROR(LoadDelegate(cc));
 #endif
@@ -365,13 +381,51 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TfLiteInferenceCalculator::InitTFLiteGPURunner() {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  // Create and bind OpenGL buffers for outputs.
+  // These buffers are created onve and later their ids are jut passed to the
+  // calculator outputs.
+
+  gpu_data_out_.resize(tflite_gpu_runner_->outputs_size());
+  for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) {
+    gpu_data_out_[i] = absl::make_unique<GPUData>();
+    ASSIGN_OR_RETURN(gpu_data_out_[i]->elements,
+                     tflite_gpu_runner_->GetOutputElements(i));
+    // Create and bind input buffer.
+    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+        gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
+  }
+  RET_CHECK_CALL(tflite_gpu_runner_->Build());
+#endif
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TfLiteInferenceCalculator::Process(CalculatorContext* cc) {
   // 1. Receive pre-processed tensor inputs.
-  if (gpu_input_) {
+  if (use_advanced_gpu_api_) {
-    // Read GPU input into SSBO.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+    RET_CHECK(input_tensors.empty());
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+        [this, &input_tensors]() -> ::mediapipe::Status {
+          for (int i = 0; i < input_tensors.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
+                input_tensors[i].id(), i));
+          }
+          for (int i = 0; i < gpu_data_out_.size(); ++i) {
+            MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
+                gpu_data_out_[i]->buffer.id(), i));
+          }
+          return ::mediapipe::OkStatus();
+        }));
+#endif
+  } else if (gpu_input_) {
+    // Read GPU input into SSBO.
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    const auto& input_tensors =
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
@@ -386,7 +440,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         }));
 #elif defined(MEDIAPIPE_IOS)
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+        cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     // Explicit copy input with conversion float 32 bits to 16 bits.
     gpu_data_in_.resize(input_tensors.size());
@@ -413,7 +467,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   } else {
     // Read CPU input into tensors.
     const auto& input_tensors =
-        cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+        cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
     RET_CHECK_GT(input_tensors.size(), 0);
     for (int i = 0; i < input_tensors.size(); ++i) {
       const TfLiteTensor* input_tensor = &input_tensors[i];
@@ -437,7 +491,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
-          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          if (use_advanced_gpu_api_) {
+            RET_CHECK(tflite_gpu_runner_->Invoke().ok());
+          } else {
+            RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          }
           return ::mediapipe::OkStatus();
         }));
 #elif defined(MEDIAPIPE_IOS)
@@ -448,7 +506,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   }
 
   // 3. Output processed tensors.
-  if (gpu_output_) {
+  if (use_advanced_gpu_api_) {
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+    auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
+    output_tensors->resize(gpu_data_out_.size());
+    for (int i = 0; i < gpu_data_out_.size(); ++i) {
+      output_tensors->at(i) = gpu_data_out_[0]->buffer.MakeRef();
+    }
+    cc->Outputs()
+        .Tag("TENSORS_GPU")
+        .Add(output_tensors.release(), cc->InputTimestamp());
+#endif
+  } else if (gpu_output_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
@@ -464,7 +533,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           return ::mediapipe::OkStatus();
         }));
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(MEDIAPIPE_IOS)
     // Output result tensors (GPU).
@@ -488,7 +557,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     [convert_command endEncoding];
     [command_buffer commit];
     cc->Outputs()
-        .Tag("TENSORS_GPU")
+        .Tag(kTensorsGpuTag)
         .Add(output_tensors.release(), cc->InputTimestamp());
 #else
     RET_CHECK_FAIL() << "GPU processing not enabled.";
@@ -501,8 +570,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
       output_tensors->emplace_back(*tensor);
     }
-    cc->Outputs().Tag("TENSORS").Add(output_tensors.release(),
+    cc->Outputs()
-                                     cc->InputTimestamp());
+        .Tag(kTensorsTag)
+        .Add(output_tensors.release(), cc->InputTimestamp());
   }
 
   return ::mediapipe::OkStatus();
@@ -513,6 +583,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     if (gpu_inference_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
+        interpreter_ = nullptr;
         delegate_ = nullptr;
         for (int i = 0; i < gpu_data_in_.size(); ++i) {
           gpu_data_in_[i].reset();
@@ -523,6 +594,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         return ::mediapipe::OkStatus();
       }));
 #elif defined(MEDIAPIPE_IOS)
+      interpreter_ = nullptr;
       delegate_ = nullptr;
       for (int i = 0; i < gpu_data_in_.size(); ++i) {
         gpu_data_in_[i].reset();
@@ -532,6 +604,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       }
 #endif
     } else {
+      interpreter_ = nullptr;
       delegate_ = nullptr;
     }
   }
@@ -554,6 +627,20 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
                       .Tag("CUSTOM_OP_RESOLVER")
                       .Get<tflite::ops::builtin::BuiltinOpResolver>();
   }
+
+#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+  if (use_advanced_gpu_api_) {
+    tflite::gpu::InferenceOptions options;
+    options.priority1 = tflite::gpu::InferencePriority::MIN_LATENCY;
+    options.priority2 = tflite::gpu::InferencePriority::AUTO;
+    options.priority3 = tflite::gpu::InferencePriority::AUTO;
+    options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED;
+    tflite_gpu_runner_ =
+        std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
+    return tflite_gpu_runner_->InitializeWithModel(model);
+  }
+#endif
+
 #if defined(MEDIAPIPE_EDGE_TPU)
   interpreter_ =
       BuildEdgeTpuInterpreter(model, &op_resolver, edgetpu_context_.get());
@@ -673,7 +760,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     const auto& input_indices = interpreter_->inputs();
     gpu_data_in_.resize(input_indices.size());
     for (int i = 0; i < input_indices.size(); ++i) {
-      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
       gpu_data_in_[i] = absl::make_unique<GPUData>();
       gpu_data_in_[i]->elements = 1;
       for (int d = 0; d < tensor->dims->size; ++d) {

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -42,7 +42,11 @@ message TfLiteInferenceCalculatorOptions {
     message TfLite {}
     // Delegate to run GPU inference depending on the device.
     // (Can use OpenGl, OpenCl, Metal depending on the device.)
-    message Gpu {}
+    message Gpu {
+      // Experimental, Android/Linux only. Use TFLite GPU delegate API2 for
+      // the NN inference.
+      optional bool use_advanced_gpu_api = 1 [default = false];
+    }
     // Android only.
     message Nnapi {}
     message Xnnpack {

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -47,10 +47,11 @@
 #endif  // iOS
 
 namespace {
-
 constexpr int kNumInputTensorsWithAnchors = 3;
 constexpr int kNumCoordsPerBox = 4;
 
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
 }  // namespace
 
 namespace mediapipe {
@@ -99,7 +100,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
                                int num_boxes, float* raw_anchors) {
   CHECK_EQ(anchors.size(), num_boxes);
   int box = 0;
-  for (auto anchor : anchors) {
+  for (const auto& anchor : anchors) {
     raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center();
     raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center();
     raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h();
@@ -200,13 +201,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
   bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("TENSORS")) {
+  if (cc->Inputs().HasTag(kTensorsTag)) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
 
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -236,7 +237,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     gpu_input_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -258,8 +259,8 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::Process(
     CalculatorContext* cc) {
-  if ((!gpu_input_ && cc->Inputs().Tag("TENSORS").IsEmpty()) ||
+  if ((!gpu_input_ && cc->Inputs().Tag(kTensorsTag).IsEmpty()) ||
-      (gpu_input_ && cc->Inputs().Tag("TENSORS_GPU").IsEmpty())) {
+      (gpu_input_ && cc->Inputs().Tag(kTensorsGpuTag).IsEmpty())) {
     return ::mediapipe::OkStatus();
   }
 
@@ -284,7 +285,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU(
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
 
   if (input_tensors.size() == 2 ||
       input_tensors.size() == kNumInputTensorsWithAnchors) {
@@ -402,7 +403,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
     CalculatorContext* cc, std::vector<Detection>* output_detections) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc,
@@ -466,7 +467,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 #elif defined(MEDIAPIPE_IOS)
 
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GpuTensor>>();
   RET_CHECK_GE(input_tensors.size(), 2);
 
   // Copy inputs.

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -145,7 +145,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
           ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
           : options_.flip_horizontally();
 
-  flip_horizontally_ =
+  flip_vertically_ =
       cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
           ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
           : options_.flip_vertically();

mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc

@@ -49,6 +49,16 @@ int NumGroups(const int size, const int group_size) {  // NOLINT
 float Clamp(float val, float min, float max) {
   return std::min(std::max(val, min), max);
 }
+
+constexpr char kTensorsTag[] = "TENSORS";
+constexpr char kTensorsGpuTag[] = "TENSORS_GPU";
+constexpr char kSizeImageTag[] = "REFERENCE_IMAGE";
+constexpr char kSizeImageGpuTag[] = "REFERENCE_IMAGE_GPU";
+constexpr char kMaskTag[] = "MASK";
+constexpr char kMaskGpuTag[] = "MASK_GPU";
+constexpr char kPrevMaskTag[] = "PREV_MASK";
+constexpr char kPrevMaskGpuTag[] = "PREV_MASK_GPU";
+
 }  // namespace
 
 namespace mediapipe {
@@ -148,39 +158,39 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   bool use_gpu = false;
 
   // Inputs CPU.
-  if (cc->Inputs().HasTag("TENSORS")) {
+  if (cc->Inputs().HasTag(kTensorsTag)) {
-    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+    cc->Inputs().Tag(kTensorsTag).Set<std::vector<TfLiteTensor>>();
   }
-  if (cc->Inputs().HasTag("PREV_MASK")) {
+  if (cc->Inputs().HasTag(kPrevMaskTag)) {
-    cc->Inputs().Tag("PREV_MASK").Set<ImageFrame>();
+    cc->Inputs().Tag(kPrevMaskTag).Set<ImageFrame>();
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
-    cc->Inputs().Tag("REFERENCE_IMAGE").Set<ImageFrame>();
+    cc->Inputs().Tag(kSizeImageTag).Set<ImageFrame>();
   }
 
   // Inputs GPU.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
-    cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GlBuffer>>();
+    cc->Inputs().Tag(kTensorsGpuTag).Set<std::vector<GlBuffer>>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("PREV_MASK_GPU")) {
+  if (cc->Inputs().HasTag(kPrevMaskGpuTag)) {
-    cc->Inputs().Tag("PREV_MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kPrevMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
-    cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Inputs().Tag(kSizeImageGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
 
   // Outputs.
-  if (cc->Outputs().HasTag("MASK")) {
+  if (cc->Outputs().HasTag(kMaskTag)) {
-    cc->Outputs().Tag("MASK").Set<ImageFrame>();
+    cc->Outputs().Tag(kMaskTag).Set<ImageFrame>();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-  if (cc->Outputs().HasTag("MASK_GPU")) {
+  if (cc->Outputs().HasTag(kMaskGpuTag)) {
-    cc->Outputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+    cc->Outputs().Tag(kMaskGpuTag).Set<mediapipe::GpuBuffer>();
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
@@ -197,7 +207,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
     CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("TENSORS_GPU")) {
+  if (cc->Inputs().HasTag(kTensorsGpuTag)) {
     use_gpu_ = true;
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
@@ -255,23 +265,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessCpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+      cc->Inputs().Tag(kTensorsTag).Get<std::vector<TfLiteTensor>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK") &&
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskTag) &&
-                             !cc->Inputs().Tag("PREV_MASK").IsEmpty();
+                             !cc->Inputs().Tag(kPrevMaskTag).IsEmpty();
   const ImageFrame placeholder;
-  const auto& input_mask = has_prev_mask
+  const auto& input_mask =
-                               ? cc->Inputs().Tag("PREV_MASK").Get<ImageFrame>()
+      has_prev_mask ? cc->Inputs().Tag(kPrevMaskTag).Get<ImageFrame>()
-                               : placeholder;
+                    : placeholder;
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE")) {
+  if (cc->Inputs().HasTag(kSizeImageTag)) {
-    const auto& input_image =
+    const auto& input_image = cc->Inputs().Tag(kSizeImageTag).Get<ImageFrame>();
-        cc->Inputs().Tag("REFERENCE_IMAGE").Get<ImageFrame>();
     output_width = input_image.Width();
     output_height = input_image.Height();
   }
@@ -353,7 +362,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
       ImageFormat::SRGBA, output_width, output_height);
   cv::Mat output_mat = formats::MatView(output_mask.get());
   large_mask_mat.copyTo(output_mat);
-  cc->Outputs().Tag("MASK").Add(output_mask.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
 
   return ::mediapipe::OkStatus();
 }
@@ -364,23 +373,23 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
 // 3. upsample small mask into output mask to be same size as input image
 ::mediapipe::Status TfLiteTensorsToSegmentationCalculator::ProcessGpu(
     CalculatorContext* cc) {
-  if (cc->Inputs().Tag("TENSORS_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Get input streams.
   const auto& input_tensors =
-      cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GlBuffer>>();
+      cc->Inputs().Tag(kTensorsGpuTag).Get<std::vector<GlBuffer>>();
-  const bool has_prev_mask = cc->Inputs().HasTag("PREV_MASK_GPU") &&
+  const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskGpuTag) &&
-                             !cc->Inputs().Tag("PREV_MASK_GPU").IsEmpty();
+                             !cc->Inputs().Tag(kPrevMaskGpuTag).IsEmpty();
   const auto& input_mask =
       has_prev_mask
-          ? cc->Inputs().Tag("PREV_MASK_GPU").Get<mediapipe::GpuBuffer>()
+          ? cc->Inputs().Tag(kPrevMaskGpuTag).Get<mediapipe::GpuBuffer>()
           : mediapipe::GpuBuffer();
   int output_width = tensor_width_, output_height = tensor_height_;
-  if (cc->Inputs().HasTag("REFERENCE_IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kSizeImageGpuTag)) {
     const auto& input_image =
-        cc->Inputs().Tag("REFERENCE_IMAGE_GPU").Get<mediapipe::GpuBuffer>();
+        cc->Inputs().Tag(kSizeImageGpuTag).Get<mediapipe::GpuBuffer>();
     output_width = input_image.width();
     output_height = input_image.height();
   }
@@ -441,7 +450,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator);
   // Send out image as GPU packet.
   auto output_image = output_texture.GetFrame<mediapipe::GpuBuffer>();
   cc->Outputs()
-      .Tag("MASK_GPU")
+      .Tag(kMaskGpuTag)
       .Add(output_image.release(), cc->InputTimestamp());
 
   // Cleanup

mediapipe/calculators/tflite/util.h

@@ -15,11 +15,11 @@
 #ifndef MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_
 #define MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_
 
-#define RET_CHECK_CALL(call)                                     \
+#define RET_CHECK_CALL(call)                               \
-  do {                                                           \
+  do {                                                     \
-    const auto status = (call);                                  \
+    const auto status = (call);                            \
-    if (ABSL_PREDICT_FALSE(!status.ok()))                        \
+    if (ABSL_PREDICT_FALSE(!status.ok()))                  \
-      return ::mediapipe::InternalError(status.error_message()); \
+      return ::mediapipe::InternalError(status.message()); \
   } while (0);
 
 #endif  // MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_

mediapipe/calculators/util/BUILD

@@ -321,7 +321,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -349,7 +349,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -926,7 +926,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -971,9 +971,9 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -55,7 +55,7 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
 // When using GPU, this color will become transparent when the calculator
 // merges the annotation overlay with the image frame. As a result, drawing in
 // this color is not supported and it should be set to something unlikely used.
-constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
+constexpr uchar kAnnotationBackgroundColor = 2;  // Grayscale value.
 }  // namespace
 
 // A calculator for rendering data on images.
@@ -491,11 +491,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     if (format != mediapipe::ImageFormat::SRGBA &&
         format != mediapipe::ImageFormat::SRGB)
       RET_CHECK_FAIL() << "Unsupported GPU input format: " << format;
-
+    image_mat = absl::make_unique<cv::Mat>(height_, width_, CV_8UC3);
-    image_mat = absl::make_unique<cv::Mat>(
+    memset(image_mat->data, kAnnotationBackgroundColor,
-        height_, width_, CV_8UC3,
+           height_ * width_ * image_mat->elemSize());
-        cv::Scalar(kAnnotationBackgroundColor[0], kAnnotationBackgroundColor[1],
-                   kAnnotationBackgroundColor[2]));
   } else {
     image_mat = absl::make_unique<cv::Mat>(
         options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
@@ -617,9 +615,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
   glUniform1i(glGetUniformLocation(program_, "overlay"), 2);
   glUniform3f(glGetUniformLocation(program_, "transparent_color"),
-              kAnnotationBackgroundColor[0] / 255.0,
+              kAnnotationBackgroundColor / 255.0,
-              kAnnotationBackgroundColor[1] / 255.0,
+              kAnnotationBackgroundColor / 255.0,
-              kAnnotationBackgroundColor[2] / 255.0);
+              kAnnotationBackgroundColor / 255.0);
 
   // Init texture for opencv rendered frame.
   const auto& input_frame =

mediapipe/calculators/util/detection_label_id_to_text_calculator.cc

@@ -71,16 +71,23 @@ REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator);
   const auto& options =
       cc->Options<::mediapipe::DetectionLabelIdToTextCalculatorOptions>();
 
-  std::string string_path;
+  if (options.has_label_map_path()) {
-  ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path()));
+    std::string string_path;
-  std::string label_map_string;
+    ASSIGN_OR_RETURN(string_path,
-  MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+                     PathToResourceAsFile(options.label_map_path()));
+    std::string label_map_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
 
-  std::istringstream stream(label_map_string);
+    std::istringstream stream(label_map_string);
-  std::string line;
+    std::string line;
-  int i = 0;
+    int i = 0;
-  while (std::getline(stream, line)) {
+    while (std::getline(stream, line)) {
-    label_map_[i++] = line;
+      label_map_[i++] = line;
+    }
+  } else {
+    for (int i = 0; i < options.label_size(); ++i) {
+      label_map_[i] = options.label(i);
+    }
   }
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/util/detection_label_id_to_text_calculator.proto

@@ -25,4 +25,10 @@ message DetectionLabelIdToTextCalculatorOptions {
 
   // Path to a label map file for getting the actual name of detected classes.
   optional string label_map_path = 1;
+
+  // Alternative way to specify label map
+  // label: "label for id 0"
+  // label: "label for id 1"
+  // ...
+  repeated string label = 2;
 }

mediapipe/calculators/util/labels_to_render_data_calculator.cc

@@ -128,16 +128,19 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator);
   } else {
     const std::vector<std::string>& label_vector =
         cc->Inputs().Tag("LABELS").Get<std::vector<std::string>>();
-    std::vector<float> score_vector;
-    if (cc->Inputs().HasTag("SCORES")) {
-      score_vector = cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
-    }
-    CHECK_EQ(label_vector.size(), score_vector.size());
     labels.resize(label_vector.size());
-    scores.resize(label_vector.size());
     for (int i = 0; i < label_vector.size(); ++i) {
       labels[i] = label_vector[i];
-      scores[i] = score_vector[i];
+    }
+
+    if (cc->Inputs().HasTag("SCORES")) {
+      std::vector<float> score_vector =
+          cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
+      CHECK_EQ(label_vector.size(), score_vector.size());
+      scores.resize(label_vector.size());
+      for (int i = 0; i < label_vector.size(); ++i) {
+        scores[i] = score_vector[i];
+      }
     }
   }
 

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -186,6 +186,7 @@ class LandmarksToRenderDataCalculator : public CalculatorBase {
 
  private:
   LandmarksToRenderDataCalculatorOptions options_;
+  std::vector<int> landmark_connections_;
 };
 REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
 
@@ -217,6 +218,14 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
   cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<LandmarksToRenderDataCalculatorOptions>();
 
+  // Parse landmarks connections to a vector.
+  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
+      << "Number of entries in landmark connections must be a multiple of 2";
+
+  for (int i = 0; i < options_.landmark_connections_size(); ++i) {
+    landmark_connections_.push_back(options_.landmark_connections(i));
+  }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -236,14 +245,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     thickness *= render_scale;
   }
 
-  // Parse landmarks connections to a vector.
-  RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0)
-      << "Number of entries in landmark connections must be a multiple of 2";
-  std::vector<int> landmark_connections;
-  for (int i = 0; i < options_.landmark_connections_size(); i += 1) {
-    landmark_connections.push_back(options_.landmark_connections(i));
-  }
-
   if (cc->Inputs().HasTag(kLandmarksTag)) {
     const LandmarkList& landmarks =
         cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
@@ -252,6 +253,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/false,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<LandmarkList, Landmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/false, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -265,15 +275,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<LandmarkList, Landmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/false,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<LandmarkList, Landmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/false, render_data.get());
-    }
   }
 
   if (cc->Inputs().HasTag(kNormLandmarksTag)) {
@@ -285,6 +286,15 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
     }
     // Only change rendering if there are actually z values other than 0.
     visualize_depth &= ((z_max - z_min) > 1e-3);
+    if (visualize_depth) {
+      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, thickness, /*normalized=*/true,
+          z_min, z_max, render_data.get());
+    } else {
+      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
+          landmarks, landmark_connections_, options_.connection_color(),
+          thickness, /*normalized=*/true, render_data.get());
+    }
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
       auto* landmark_data_render = AddPointRenderData(
@@ -298,15 +308,6 @@ REGISTER_CALCULATOR(LandmarksToRenderDataCalculator);
       landmark_data->set_x(landmark.x());
       landmark_data->set_y(landmark.y());
     }
-    if (visualize_depth) {
-      AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, thickness, /*normalized=*/true,
-          z_min, z_max, render_data.get());
-    } else {
-      AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
-          landmarks, landmark_connections, options_.connection_color(),
-          thickness, /*normalized=*/true, render_data.get());
-    }
   }
 
   cc->Outputs()

mediapipe/calculators/util/local_file_contents_calculator.cc

@@ -16,34 +16,80 @@
 #include <string>
 
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
 
 namespace mediapipe {
+
+namespace {
+
+constexpr char kFilePathTag[] = "FILE_PATH";
+constexpr char kContentsTag[] = "CONTENTS";
+
+}  // namespace
+
 // The calculator takes the path to the local file as an input side packet and
 // outputs the contents of that file.
 //
+// NOTE: file loading can be batched by providing multiple input/output side
+// packets.
+//
 // Example config:
 // node {
 //   calculator: "LocalFileContentsCalculator"
 //   input_side_packet: "FILE_PATH:file_path"
 //   output_side_packet: "CONTENTS:contents"
 // }
+//
+// node {
+//   calculator: "LocalFileContentsCalculator"
+//   input_side_packet: "FILE_PATH:0:file_path1"
+//   input_side_packet: "FILE_PATH:1:file_path2"
+//   ...
+//   output_side_packet: "CONTENTS:0:contents1"
+//   output_side_packet: "CONTENTS:1:contents2"
+//   ...
+// }
 class LocalFileContentsCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->InputSidePackets().Tag("FILE_PATH").Set<std::string>();
+    RET_CHECK(cc->InputSidePackets().HasTag(kFilePathTag))
-    cc->OutputSidePackets().Tag("CONTENTS").Set<std::string>();
+        << "Missing PATH input side packet(s)";
+    RET_CHECK(cc->OutputSidePackets().HasTag(kContentsTag))
+        << "Missing CONTENTS output side packet(s)";
+
+    RET_CHECK_EQ(cc->InputSidePackets().NumEntries(kFilePathTag),
+                 cc->OutputSidePackets().NumEntries(kContentsTag))
+        << "Same number of input streams and output streams is required.";
+
+    for (CollectionItemId id = cc->InputSidePackets().BeginId(kFilePathTag);
+         id != cc->InputSidePackets().EndId(kFilePathTag); ++id) {
+      cc->InputSidePackets().Get(id).Set<std::string>();
+    }
+
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kContentsTag);
+         id != cc->OutputSidePackets().EndId(kContentsTag); ++id) {
+      cc->OutputSidePackets().Get(id).Set<std::string>();
+    }
+
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
-    std::string contents;
+    CollectionItemId input_id = cc->InputSidePackets().BeginId(kFilePathTag);
-    MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
+    CollectionItemId output_id = cc->OutputSidePackets().BeginId(kContentsTag);
-        cc->InputSidePackets().Tag("FILE_PATH").Get<std::string>(), &contents));
+    // Number of inputs and outpus is the same according to the contract.
-    cc->OutputSidePackets()
+    for (; input_id != cc->InputSidePackets().EndId(kFilePathTag);
-        .Tag("CONTENTS")
+         ++input_id, ++output_id) {
-        .Set(MakePacket<std::string>(std::move(contents)));
+      std::string file_path =
+          cc->InputSidePackets().Get(input_id).Get<std::string>();
+      ASSIGN_OR_RETURN(file_path, PathToResourceAsFile(file_path));
+
+      std::string contents;
+      MP_RETURN_IF_ERROR(GetResourceContents(file_path, &contents));
+      cc->OutputSidePackets().Get(output_id).Set(
+          MakePacket<std::string>(std::move(contents)));
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
+
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -76,14 +78,16 @@ void AddTimedBoxProtoToRenderData(
     RenderAnnotation::Text* text = label_annotation->mutable_text();
     text->set_display_text(box_proto.label());
     text->set_normalized(true);
-    constexpr float text_left_start = 0.3f;
+    constexpr float text_left_start = 0.2f;
     text->set_left((1.0f - text_left_start) * box_proto.left() +
                    text_left_start * box_proto.right());
     constexpr float text_baseline = 0.6f;
     text->set_baseline(text_baseline * box_proto.bottom() +
                        (1.0f - text_baseline) * box_proto.top());
-    constexpr float text_height = 0.2f;
+    constexpr float text_height = 0.1f;
-    text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height);
+    text->set_font_height(std::min(box_proto.bottom() - box_proto.top(),
+                                   box_proto.right() - box_proto.left()) *
+                          text_height);
   }
 }
 

mediapipe/calculators/video/BUILD

@@ -65,6 +65,26 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "tracked_detection_manager_calculator_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_proto",
+    ],
+)
+
+proto_library(
+    name = "box_detector_calculator_proto",
+    srcs = ["box_detector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:box_detector_proto",
+    ],
+)
+
 proto_library(
     name = "video_pre_stream_calculator_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -107,6 +127,28 @@ mediapipe_cc_proto_library(
     deps = [":box_tracker_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tracked_detection_manager_calculator_cc_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":tracked_detection_manager_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "box_detector_calculator_cc_proto",
+    srcs = ["box_detector_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:box_detector_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":box_detector_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "video_pre_stream_calculator_cc_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -279,11 +321,54 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "box_detector_calculator",
+    srcs = ["box_detector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":box_detector_calculator_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",  # fixdeps: keep -- required for exobazel build.
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_detector",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:flow_packager_cc_proto",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+    ] + select({
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:ios": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tracked_detection_manager_calculator",
     srcs = ["tracked_detection_manager_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":tracked_detection_manager_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",

mediapipe/calculators/video/box_detector_calculator.cc

@@ -0,0 +1,393 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <memory>
+#include <unordered_set>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/numbers.h"
+#include "mediapipe/calculators/video/box_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
+#include "mediapipe/util/tracking/box_detector.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/flow_packager.pb.h"
+#include "mediapipe/util/tracking/tracking.h"
+#include "mediapipe/util/tracking/tracking_visualization_utilities.h"
+
+#if defined(MEDIAPIPE_MOBILE)
+#include "mediapipe/util/android/file/base/file.h"
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+// A calculator to detect reappeared box positions from single frame.
+//
+// Input stream:
+//   TRACKING: Input tracking data (proto TrackingData) containing features and
+//             descriptors.
+//   VIDEO:    Optional input video stream tracked boxes are rendered over
+//             (Required if VIZ is specified).
+//   FEATURES: Input feature points (std::vector<cv::KeyPoint>) in the original
+//             pixel space.
+//   DESCRIPTORS: Input feature descriptors (std::vector<float>). Actual feature
+//             dimension needs to be specified in detector_options.
+//   IMAGE_SIZE: Input image dimension.
+//   TRACKED_BOXES : input box tracking result (proto TimedBoxProtoList) from
+//             BoxTrackerCalculator.
+//   ADD_INDEX: Optional std::string containing binary format proto of type
+//             BoxDetectorIndex. Used for adding target index to the detector
+//             search index during runtime.
+//   CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended
+//             to be used with SyncSetInputStreamHandler.
+//   REACQ_SWITCH: Optional bool for swithcing on and off reacquisition
+//             functionality. User should initialize a graph with box detector
+//             calculator and be able to switch it on and off in runtime.
+//
+// Output streams:
+//   VIZ:   Optional output video stream with rendered box positions
+//          (requires VIDEO to be present)
+//   BOXES: Optional output stream of type TimedBoxProtoList for each lost box.
+//
+// Imput side packets:
+//   INDEX_PROTO_STRING: Optional std::string containing binary format proto of
+//   type
+//                       BoxDetectorIndex. Used for initializing box_detector
+//                       with predefined template images.
+//   FRAME_ALIGNMENT:    Optional integer to indicate alignment_boundary for
+//                       outputing ImageFrame in "VIZ" stream.
+//                       Set to ImageFrame::kDefaultAlignmentBoundary for
+//                       offline pipeline to be compatible with FFmpeg.
+//                       Set to ImageFrame::kGlDefaultAlignmentBoundary for Apps
+//                       to be compatible with GL renderer.
+//   OUTPUT_INDEX_FILENAME: File path to the output index file.
+
+class BoxDetectorCalculator : public CalculatorBase {
+ public:
+  ~BoxDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  BoxDetectorCalculatorOptions options_;
+  std::unique_ptr<BoxDetectorInterface> box_detector_;
+  bool detector_switch_ = true;
+  uint32 frame_alignment_ = ImageFrame::kDefaultAlignmentBoundary;
+  bool write_index_ = false;
+  int box_id_ = 0;
+};
+
+REGISTER_CALCULATOR(BoxDetectorCalculator);
+
+::mediapipe::Status BoxDetectorCalculator::GetContract(CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("TRACKING")) {
+    cc->Inputs().Tag("TRACKING").Set<TrackingData>();
+  }
+
+  if (cc->Inputs().HasTag("TRACKED_BOXES")) {
+    cc->Inputs().Tag("TRACKED_BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Inputs().HasTag("VIDEO")) {
+    cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  }
+
+  if (cc->Inputs().HasTag("FEATURES")) {
+    RET_CHECK(cc->Inputs().HasTag("DESCRIPTORS"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+
+  if (cc->Inputs().HasTag("DESCRIPTORS")) {
+    RET_CHECK(cc->Inputs().HasTag("FEATURES"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("DESCRIPTORS").Set<std::vector<float>>();
+  }
+
+  if (cc->Inputs().HasTag("IMAGE_SIZE")) {
+    cc->Inputs().Tag("IMAGE_SIZE").Set<std::pair<int, int>>();
+  }
+
+  if (cc->Inputs().HasTag("ADD_INDEX")) {
+    cc->Inputs().Tag("ADD_INDEX").Set<std::string>();
+  }
+
+  if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) {
+    cc->Inputs().Tag("CANCEL_OBJECT_ID").Set<int>();
+  }
+
+  if (cc->Inputs().HasTag("REACQ_SWITCH")) {
+    cc->Inputs().Tag("REACQ_SWITCH").Set<bool>();
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    RET_CHECK(cc->Inputs().HasTag("VIDEO"))
+        << "Output stream VIZ requires VIDEO to be present.";
+    cc->Outputs().Tag("VIZ").Set<ImageFrame>();
+  }
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    cc->InputSidePackets().Tag("INDEX_PROTO_STRING").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Set<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<BoxDetectorCalculatorOptions>();
+  box_detector_ = BoxDetectorInterface::Create(options_.detector_options());
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(cc->InputSidePackets()
+                                              .Tag("INDEX_PROTO_STRING")
+                                              .Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  for (const auto& filename : options_.index_proto_filename()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(filename));
+    std::string index_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string));
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(index_string)) {
+      LOG(FATAL)
+          << "failed to parse BoxDetectorIndex from index_proto_filename";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    write_index_ = true;
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    frame_alignment_ = cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Get<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp timestamp = cc->InputTimestamp();
+  const int64 timestamp_msec = timestamp.Value() / 1000;
+
+  InputStream* cancel_object_id_stream =
+      cc->Inputs().HasTag("CANCEL_OBJECT_ID")
+          ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID"))
+          : nullptr;
+  if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) {
+    const int cancel_object_id = cancel_object_id_stream->Get<int>();
+    box_detector_->CancelBoxDetection(cancel_object_id);
+  }
+
+  InputStream* add_index_stream = cc->Inputs().HasTag("ADD_INDEX")
+                                      ? &(cc->Inputs().Tag("ADD_INDEX"))
+                                      : nullptr;
+  if (add_index_stream && !add_index_stream->IsEmpty()) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(
+            add_index_stream->Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  InputStream* reacq_switch_stream = cc->Inputs().HasTag("REACQ_SWITCH")
+                                         ? &(cc->Inputs().Tag("REACQ_SWITCH"))
+                                         : nullptr;
+  if (reacq_switch_stream && !reacq_switch_stream->IsEmpty()) {
+    detector_switch_ = reacq_switch_stream->Get<bool>();
+  }
+
+  if (!detector_switch_) {
+    return ::mediapipe::OkStatus();
+  }
+
+  InputStream* track_stream = cc->Inputs().HasTag("TRACKING")
+                                  ? &(cc->Inputs().Tag("TRACKING"))
+                                  : nullptr;
+  InputStream* video_stream =
+      cc->Inputs().HasTag("VIDEO") ? &(cc->Inputs().Tag("VIDEO")) : nullptr;
+  InputStream* feature_stream = cc->Inputs().HasTag("FEATURES")
+                                    ? &(cc->Inputs().Tag("FEATURES"))
+                                    : nullptr;
+  InputStream* descriptor_stream = cc->Inputs().HasTag("DESCRIPTORS")
+                                       ? &(cc->Inputs().Tag("DESCRIPTORS"))
+                                       : nullptr;
+
+  CHECK(track_stream != nullptr || video_stream != nullptr ||
+        (feature_stream != nullptr && descriptor_stream != nullptr))
+      << "One and only one of {tracking_data, input image frame, "
+         "feature/descriptor} need to be valid.";
+
+  InputStream* tracked_boxes_stream = cc->Inputs().HasTag("TRACKED_BOXES")
+                                          ? &(cc->Inputs().Tag("TRACKED_BOXES"))
+                                          : nullptr;
+  std::unique_ptr<TimedBoxProtoList> detected_boxes(new TimedBoxProtoList());
+
+  if (track_stream != nullptr) {
+    // Detect from tracking data
+    if (track_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const TrackingData& tracking_data = track_stream->Get<TrackingData>();
+
+    CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed.";
+
+    const TimedBoxProtoList tracked_boxes =
+        tracked_boxes_stream->Get<TimedBoxProtoList>();
+
+    box_detector_->DetectAndAddBox(tracking_data, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else if (video_stream != nullptr) {
+    // Detect from input frame
+    if (video_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    }
+
+    // Just directly pass along the image frame data as-is for detection; we
+    // don't need to worry about conforming to a specific alignment here.
+    const cv::Mat input_view =
+        formats::MatView(&video_stream->Get<ImageFrame>());
+    box_detector_->DetectAndAddBox(input_view, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else {
+    if (feature_stream->IsEmpty() || descriptor_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const auto& image_size =
+        cc->Inputs().Tag("IMAGE_SIZE").Get<std::pair<int, int>>();
+    float inv_scale = 1.0f / std::max(image_size.first, image_size.second);
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    } else if (write_index_) {
+      auto* box_ptr = tracked_boxes.add_box();
+      box_ptr->set_id(box_id_);
+      box_ptr->set_reacquisition(true);
+      box_ptr->set_aspect_ratio((float)image_size.first /
+                                (float)image_size.second);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      ++box_id_;
+    }
+
+    const auto& features = feature_stream->Get<std::vector<cv::KeyPoint>>();
+    const int feature_size = features.size();
+    std::vector<Vector2_f> features_vec(feature_size);
+
+    const auto& descriptors = descriptor_stream->Get<std::vector<float>>();
+    const int dims = options_.detector_options().descriptor_dims();
+    CHECK_GE(descriptors.size(), feature_size * dims);
+    cv::Mat descriptors_mat(feature_size, dims, CV_32F);
+    for (int j = 0; j < feature_size; ++j) {
+      features_vec[j].Set(features[j].pt.x * inv_scale,
+                          features[j].pt.y * inv_scale);
+      for (int i = 0; i < dims; ++i) {
+        descriptors_mat.at<float>(j, i) = descriptors[j * dims + i];
+      }
+    }
+
+    box_detector_->DetectAndAddBoxFromFeatures(
+        features_vec, descriptors_mat, tracked_boxes, timestamp_msec,
+        image_size.first * inv_scale, image_size.second * inv_scale,
+        detected_boxes.get());
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    cv::Mat viz_view;
+    std::unique_ptr<ImageFrame> viz_frame;
+    if (video_stream != nullptr && !video_stream->IsEmpty()) {
+      viz_frame = absl::make_unique<ImageFrame>();
+      viz_frame->CopyFrom(video_stream->Get<ImageFrame>(), frame_alignment_);
+      viz_view = formats::MatView(viz_frame.get());
+    }
+    for (const auto& box : detected_boxes->box()) {
+      RenderBox(box, &viz_view);
+    }
+    cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Add(detected_boxes.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Close(CalculatorContext* cc) {
+  if (write_index_) {
+    BoxDetectorIndex index = box_detector_->ObtainBoxDetectorIndex();
+    MEDIAPIPE_CHECK_OK(mediapipe::file::SetContents(
+        cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Get<std::string>(),
+        index.SerializeAsString()));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/box_detector_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/box_detector.proto";
+
+message BoxDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional BoxDetectorCalculatorOptions ext = 289746530;
+  }
+
+  optional BoxDetectorOptions detector_options = 1;
+
+  // File path to the template index files.
+  repeated string index_proto_filename = 2;
+}

mediapipe/calculators/video/tracked_detection_manager_calculator.cc

@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "absl/container/node_hash_map.h"
+#include "mediapipe/calculators/video/tracked_detection_manager_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
@@ -139,6 +140,7 @@ Detection GetAxisAlignedDetectionFromTrackedDetection(
 class TrackedDetectionManagerCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
 
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
@@ -184,6 +186,15 @@ REGISTER_CALCULATOR(TrackedDetectionManagerCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TrackedDetectionManagerCalculator::Open(
+    CalculatorContext* cc) {
+  mediapipe::TrackedDetectionManagerCalculatorOptions options =
+      cc->Options<mediapipe::TrackedDetectionManagerCalculatorOptions>();
+  tracked_detection_manager_.SetConfig(
+      options.tracked_detection_manager_options());
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TrackedDetectionManagerCalculator::Process(
     CalculatorContext* cc) {
   if (cc->Inputs().HasTag("TRACKING_BOXES")) {

mediapipe/calculators/video/tracked_detection_manager_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/tracked_detection_manager_config.proto";
+
+message TrackedDetectionManagerCalculatorOptions {
+  extend CalculatorOptions {
+    optional TrackedDetectionManagerCalculatorOptions ext = 301970230;
+  }
+
+  optional TrackedDetectionManagerConfig tracked_detection_manager_options = 1;
+}

mediapipe/docs/android_archive_library.md

@@ -32,13 +32,14 @@ project.
 2.  Run the Bazel build command to generate the AAR.
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a //path/to/the/aar/build/file:aar_name
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+        //path/to/the/aar/build/file:aar_name
     ```
 
     For the face detection AAR target we made in the step 1, run:
 
     ```bash
-    bazel build -c opt --fat_apk_cpu=arm64-v8a,armeabi-v7a \
+    bazel build -c opt --host_crosstool_top=@bazel_tools//tools/cpp:toolchain --fat_apk_cpu=arm64-v8a,armeabi-v7a \
         //mediapipe/examples/android/src/java/com/google/mediapipe/apps/aar_example:mp_face_detection_aar
 
     # It should print:

mediapipe/docs/examples.md

@@ -73,6 +73,18 @@ can be easily adapted to run on CPU v.s. GPU.
 *   [Android](./face_detection_mobile_cpu.md)
 *   [iOS](./face_detection_mobile_cpu.md)
 
+### Face Mesh with GPU
+
+[Face Mesh with GPU](./face_mesh_mobile_gpu.md) illustrates how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on mobile devices, utilizing GPU acceleration. The pipeline is based on
+["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724),
+and details of the underlying ML models are described in the
+[model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view).
+
+*   [Android](./face_mesh_mobile_gpu.md)
+*   [iOS](./face_mesh_mobile_gpu.md)
+
 ### Hand Detection with GPU
 
 [Hand Detection with GPU](./hand_detection_mobile_gpu.md) illustrates how to use
@@ -84,7 +96,7 @@ MediaPipe with a TFLite model for hand detection in a GPU-accelerated pipeline.
 ### Hand Tracking with GPU
 
 [Hand Tracking with GPU](./hand_tracking_mobile_gpu.md) illustrates how to use
-MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
+MediaPipe with TFLite models for hand tracking in a GPU-accelerated pipeline.
 
 *   [Android](./hand_tracking_mobile_gpu.md)
 *   [iOS](./hand_tracking_mobile_gpu.md)
@@ -92,7 +104,7 @@ MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
 ### Multi-Hand Tracking with GPU
 
 [Multi-Hand Tracking with GPU](./multi_hand_tracking_mobile_gpu.md) illustrates
-how to use MediaPipe with a TFLite model for multi-hand tracking in a
+how to use MediaPipe with TFLite models for multi-hand tracking in a
 GPU-accelerated pipeline.
 
 *   [Android](./multi_hand_tracking_mobile_gpu.md)
@@ -109,6 +121,14 @@ and model details are described in the
 
 *   [Android](./hair_segmentation_mobile_gpu.md)
 
+### Template Matching using KNIFT with CPU
+
+[Template Matching using KNIFT on Mobile](./template_matching_mobile_cpu.md)
+shows how to use MediaPipe with TFLite model for template matching using Knift
+on mobile using CPU.
+
+*   [Android](./template_matching_mobile_cpu.md)
+
 ## Desktop
 
 ### Hello World for C++
@@ -150,11 +170,19 @@ GPU with live video from a webcam.
 *   [Desktop GPU](./face_detection_desktop.md)
 *   [Desktop CPU](./face_detection_desktop.md)
 
+### Face Mesh on Desktop with Webcam
+
+[Face Mesh on Desktop with Webcam](./face_mesh_desktop.md) shows how to run the
+MediaPipe Face Mesh pipeline to perform 3D face landmark estimation in real-time
+on desktop with webcam input.
+
+*   [Desktop GPU](./face_mesh_desktop.md)
+*   [Desktop CPU](./face_mesh_desktop.md)
 
 ### Hand Tracking on Desktop with Webcam
 
 [Hand Tracking on Desktop with Webcam](./hand_tracking_desktop.md) shows how to
-use MediaPipe with a TFLite model for hand tracking on desktop using CPU or GPU
+use MediaPipe with TFLite models for hand tracking on desktop using CPU or GPU
 with live video from a webcam.
 
 *   [Desktop GPU](./hand_tracking_desktop.md)
@@ -163,8 +191,8 @@ with live video from a webcam.
 ### Multi-Hand Tracking on Desktop with Webcam
 
 [Multi-Hand Tracking on Desktop with Webcam](./multi_hand_tracking_desktop.md)
-shows how to use MediaPipe with a TFLite model for multi-hand tracking on
+shows how to use MediaPipe with TFLite models for multi-hand tracking on desktop
-desktop using CPU or GPU with live video from a webcam.
+using CPU or GPU with live video from a webcam.
 
 *   [Desktop GPU](./multi_hand_tracking_desktop.md)
 *   [Desktop CPU](./multi_hand_tracking_desktop.md)
@@ -177,7 +205,7 @@ GPU with live video from a webcam.
 
 *   [Desktop GPU](./hair_segmentation_desktop.md)
 
-## Google Coral (machine learning acceleration with Google EdgeTPU)
+## Google Coral (ML acceleration with Google EdgeTPU)
 
 Below are code samples on how to run MediaPipe on Google Coral Dev Board.
 

mediapipe/docs/face_detection_desktop.md

@@ -51,7 +51,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/face_detection:face_detection_gpu
 
 # It should print:

mediapipe/docs/face_detection_mobile_cpu.md

@@ -4,6 +4,8 @@ This doc focuses on the
 [example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt)
 that performs face detection with TensorFlow Lite on CPU.
 
+![face_detection_android_gpu_gif](images/mobile/face_detection_android_gpu.gif)
+
 ## Android
 
 [Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu)

mediapipe/docs/face_mesh_desktop.md

@@ -0,0 +1,58 @@
+## Face Mesh on Desktop with Webcam
+
+This doc focuses on running the **MediaPipe Face Mesh** pipeline to perform 3D
+face landmark estimation in real-time on desktop with webcam input. The pipeline
+internally incorporates TensorFlow Lite models. To know more about the models,
+please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+Moreover, if you are interested in running the same pipeline on Android/iOS,
+please see [Face Mesh on Android/iOS](face_mesh_mobile_gpu.md).
+
+-   [Face Mesh on Desktop with Webcam (CPU)](#face-mesh-on-desktop-with-webcam-cpu)
+
+-   [Face Mesh on Desktop with Webcam (GPU)](#face-mesh-on-desktop-with-webcam-gpu)
+
+Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
+see
+[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
+
+Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+
+### Face Mesh on Desktop with Webcam (CPU)
+
+To build and run Face Mesh on desktop with webcam (CPU), run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_cpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_cpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_cpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt
+```
+
+### Face Mesh on Desktop with Webcam (GPU)
+
+Note: please first [check that your GPU is supported](gpu.md#desktop-gpu-linux).
+
+To build and run Face Mesh on desktop with webcam (GPU), run:
+
+```bash
+# This works only for linux currently
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
+    mediapipe/examples/desktop/face_mesh:face_mesh_gpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/face_mesh:face_mesh_gpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu
+
+# This will open up your webcam as long as it is connected. Errors are likely
+# due to your webcam being not accessible, or GPU drivers not setup properly.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/face_mesh/face_mesh_gpu \
+    --calculator_graph_config_file=mediapipe/graphs/face_mesh/face_mesh_desktop_live_gpu.pbtxt
+```

mediapipe/docs/face_mesh_mobile_gpu.md

@@ -0,0 +1,90 @@
+# Face Mesh (GPU)
+
+This example focuses on running the **MediaPipe Face Mesh** pipeline on mobile
+devices to perform 3D face landmark estimation in real-time, utilizing GPU
+acceleration. The pipeline internally incorporates TensorFlow Lite models. To
+know more about the models, please refer to the model
+[README file](https://github.com/google/mediapipe/tree/master/mediapipe/models/README.md#face-mesh).
+The pipeline is related to the
+[face detection example](./face_detection_mobile_gpu.md) as it internally
+utilizes face detection and performs landmark estimation only within the
+detected region.
+
+![face_mesh_android_gpu.gif](images/mobile/face_mesh_android_gpu.gif)
+
+**MediaPipe Face Mesh** generates 468 3D face landmarks in real-time on mobile
+devices. In the visualization above, the red dots represent the landmarks, and
+the green lines connecting landmarks illustrate the contours around the eyes,
+eyebrows, lips and the entire face.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu)
+
+A prebuilt arm64 APK can be
+[downloaded here](https://drive.google.com/open?id=1pUmd7CXCL_onYMbsZo5p91cH0oNnR4gi).
+
+To build the app yourself, run:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/facemeshgpu.apk
+```
+
+## iOS
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu).
+
+See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+examples and generating an Xcode project. This will be the FaceMeshGpuApp
+target.
+
+To build on the command line:
+
+```bash
+bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp
+```
+
+## Graph
+
+The face mesh [main graph](#main-graph) utilizes a
+[face landmark subgraph](#face-landmark-subgraph) from the
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark),
+and renders using a dedicated [face renderer subgraph](#face-renderer-subgraph).
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+![face_mesh_mobile_graph](images/mobile/face_mesh_mobile.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
+
+### Face Landmark Subgraph
+
+The
+[face landmark module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark)
+contains several subgraphs that can be used to detect and track face landmarks.
+In particular, in this example the
+[FaceLandmarkFrontGPU](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+subgraph, suitable for images from front-facing cameras (i.e., selfie images)
+and utilizing GPU acceleration, is selected.
+
+![face_landmark_front_gpu_subgraph](images/mobile/face_landmark_front_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+
+### Face Renderer Subgraph
+
+![face_renderer_gpu_subgraph](images/mobile/face_renderer_gpu_subgraph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/subgraphs/face_renderer_gpu.pbtxt)

mediapipe/docs/gpu.md

@@ -190,10 +190,10 @@ within the MediaPipe framework:
 bazel build --define MEDIAPIPE_DISABLE_GPU=1  <my-target>
 
 # to enable full GPU support (OpenGL ES 3.1+ & Metal)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS  <my-target>
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 <my-target>
 
 # to enable only OpenGL ES 3.0 and below (no GLES 3.1+ features)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
 ```
 
 Note *MEDIAPIPE_DISABLE_GL_COMPUTE* is automatically defined on all Apple

mediapipe/docs/hair_segmentation_desktop.md

@@ -26,7 +26,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu
 
 # It should print:

mediapipe/docs/hand_tracking_desktop.md

@@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
 
 # It should print:

mediapipe/docs/install.md

@@ -40,12 +40,11 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 1.0.0 and 1.2.1).
+2.  Install Bazel.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    to install Bazel 2.0 or higher.
-    yet.
 
 3.  Install OpenCV and FFmpeg.
 
@@ -111,7 +110,7 @@ To build and run iOS apps:
     # To compile with GPU support, replace
     --define MEDIAPIPE_DISABLE_GPU=1
     # with
-    --copt -DMESA_EGL_NO_X11_HEADERS
+    --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11
     # when building GPU examples.
     ```
 
@@ -125,7 +124,7 @@ To build and run iOS apps:
         mediapipe/examples/desktop/hello_world:hello_world
 
     # If you are running on Linux desktop with GPU support enabled (via mesa drivers)
-    $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS \
+    $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
         mediapipe/examples/desktop/hello_world:hello_world
 
     # Should print:
@@ -152,12 +151,11 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 1.0.0 and 1.2.1).
+2.  Install Bazel.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    to install Bazel 2.0 or higher.
-    yet.
 
 3.  Install OpenCV.
 
@@ -241,23 +239,18 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-3.  Install Bazel (version between 1.0.0 and 1.1.0).
+3.  Install Bazel.
 
-    Option 1. Use package manager tool to install Bazel 1.1.0
+    Option 1. Use package manager tool to install Bazel
 
     ```bash
-    # If Bazel 1.1.0+ was installed.
+    $ brew install bazel
-    $ brew uninstall bazel
-    # Install Bazel 1.1.0
-    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb
-    $ brew link bazel
     # Run 'bazel version' to check version of bazel
     ```
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install any version of Bazel manually. Note that MediaPipe doesn't
+    to install Bazel 2.0 or higher.
-    support Bazel 1.1.0+ on macOS yet.
 
 4.  Install OpenCV and FFmpeg.
 
@@ -391,18 +384,18 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel (version between 1.0.0 and 1.2.1).
+5.  Install Bazel.
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
+    https://storage.googleapis.com/bazel/2.0.0/release/bazel-2.0.0-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/1.0.0 && \
+    sudo mkdir -p /usr/local/bazel/2.0.0 && \
-    chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
+    chmod 755 bazel-2.0.0-installer-linux-x86_64.sh && \
-    sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
+    sudo ./bazel-2.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/2.0.0 && \
-    source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
+    source /usr/local/bazel/2.0.0/lib/bazel/bin/bazel-complete.bash
 
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
+    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/2.0.0/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
+    alias bazel='/usr/local/bazel/2.0.0/lib/bazel/bin/bazel'
     ```
 
 6.  Checkout MediaPipe repository.

mediapipe/docs/media_sequence.md

@@ -89,11 +89,11 @@ process new data sets, in the documentation of
        dataset = d.as_dataset('test')
        # implement additional processing and batching here
        dataset_output = dataset.make_one_shot_iterator().get_next()
-       images = dataset_output=['images']
+       images = dataset_output['images']
-       labels = dataset_output=['labels']
+       labels = dataset_output['labels']
 
        with tf.Session() as sess:
-         images_, labels_ = sess.run(images, labels)
+         images_, labels_ = sess.run([images, labels])
        print('The shape of images_ is %s' % str(images_.shape))
        print('The shape of labels_ is %s' % str(labels_.shape))
     ```

mediapipe/docs/mediapipe_ios_setup.md

@@ -76,6 +76,10 @@ in the next section.
 
 2.  Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app.
 
+    Important: If Tulsi displays an error saying "Bazel could not be found",
+    press the "Bazel..." button in the Packages tab and select the `bazel`
+    executable in your homebrew `/bin/` directory.
+
 3.  Select the MediaPipe config in the Configs tab, then press the Generate
     button below. You will be asked for a location to save the Xcode project.
     Once the project is generated, it will be opened in Xcode.

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu
 
 # It should print:

mediapipe/docs/object_detection_desktop.md

@@ -12,7 +12,7 @@ We show the object detection demo with both TensorFlow model and TensorFlow Lite
 
 -   [TensorFlow Object Detection Demo](#tensorflow-object-detection-demo)
 -   [TensorFlow Lite Object Detection Demo](#tensorflow-lite-object-detection-demo)
--   [TensorFlow Lite Object Detection Demo with Webcam (CPU)](#tensorflow-lite-object-detection-demo)
+-   [TensorFlow Lite Object Detection Demo with Webcam (CPU)](#tensorflow-lite-object-detection-demo-with-webcam-cpu)
 
 Note: If MediaPipe depends on OpenCV 2, please see the [known issues with OpenCV 2](#known-issues-with-opencv-2) section.
 

mediapipe/docs/template_matching_desktop_cpu.md

@@ -0,0 +1,31 @@
+# Template Matching using KNIFT on Desktop
+
+This doc focuses on the
+[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_desktop.pbtxt)
+that performs template matching with KNIFT (Keypoint Neural Invariant Feature
+Transform) on desktop CPU.
+
+If you are interested in more detail about KNIFT or running the example on
+mobile, please see
+[Template Matching using KNIFT on Mobile (CPU)](template_matching_mobile_cpu.md).
+
+To build the desktop app, run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/template_matching:template_matching_tflite
+```
+
+To run the desktop app, please specify a template index file
+([example](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)) and a
+video to be matched. For how to build your own index file, please see
+[here](template_matching_mobile_cpu.md#build-index-file).
+
+```bash
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/template_matching/template_matching_tflite \
+    --calculator_graph_config_file=mediapipe/graphs/template_matching/template_matching_desktop.pbtxt --input_side_packets="input_video_path=<input video path>,output_video_path=<output video path>"
+```
+
+## Graph
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_desktop.pbtxt)

mediapipe/docs/template_matching_mobile_cpu.md

@@ -0,0 +1,94 @@
+# Template Matching using KNIFT on Mobile (CPU)
+
+This doc focuses on the
+[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt)
+that performs template matching with KNIFT (Keypoint Neural Invariant Feature
+Transform) on mobile CPU.
+
+![template_matching_mobile_cpu.gif](images/mobile/template_matching_android_cpu.gif)
+
+In the visualization above, the green dots represent detected keypoints on each
+frame and the red box represents the targets matched by templates using KNIFT
+features (see also [model card](https://mediapipe.page.link/knift-mc)). For more
+information, please see
+[Google Developers Blog](https://mediapipe.page.link/knift-blog).
+
+## Build Index Files
+
+In MediaPipe, we've already provided a file in
+[knift_index.pb](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb),
+pre-computed from the 3 template images (of USD bills) shown below. If you'd
+like to use your own template images, please follow the steps below, or
+otherwise you can jump directly to [Android](#android).
+
+![template_matching_mobile_template.jpg](images/mobile/template_matching_mobile_template.jpg)
+
+### Step 1:
+
+Put all template images in a single directory.
+
+### Step 2:
+
+To build the index file for all templates in the directory, run:
+
+```bash
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/template_matching:template_matching_tflite
+$ bazel-bin/mediapipe/examples/desktop/template_matching/template_matching_tflite \
+    --calculator_graph_config_file=mediapipe/graphs/template_matching/index_building.pbtxt \
+    --input_side_packets="file_directory=<template image directory>,file_suffix='png',output_index_filename=<output index filename>"
+```
+
+The output index file includes the extracted KNIFT features.
+
+### Step 3:
+
+Replace
+[mediapipe/models/knift_index.pb](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_index.pb)
+with the index file you generated, and update
+[mediapipe/models/knift_labelmap.txt](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_labelmap.txt)
+with your own template names.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu)
+
+A prebuilt arm64 APK can be
+[downloaded here](https://drive.google.com/open?id=1tSWRfes9rAM4NrzmJBplguNQQvaeBZSa).
+
+To build and install the app yourself, run:
+
+Note: MediaPipe uses OpenCV 3 by default. However, because of
+[issues](https://github.com/opencv/opencv/issues/11488) between NDK 17+ and
+OpenCV 3 when using
+[knnMatch](https://docs.opencv.org/3.4/db/d39/classcv_1_1DescriptorMatcher.html#a378f35c9b1a5dfa4022839a45cdf0e89),
+please use the following commands to temporarily switch to OpenCV 4 for the
+template matching exmaple on Android, and switch back to OpenCV 3 afterwards.
+
+```bash
+# Switch to OpenCV 4
+sed -i -e 's:3.4.3/opencv-3.4.3:4.0.1/opencv-4.0.1:g' WORKSPACE
+sed -i -e 's:libopencv_java3:libopencv_java4:g' third_party/opencv_android.BUILD
+
+# Build and install app
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu:templatematchingcpu
+adb install -r bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/templatematchingcpu.apk
+
+# Switch back to OpenCV 3
+sed -i -e 's:4.0.1/opencv-4.0.1:3.4.3/opencv-3.4.3:g' WORKSPACE
+sed -i -e 's:libopencv_java4:libopencv_java3:g' third_party/opencv_android.BUILD
+```
+
+## Use XNNPACK Delegate
+
+The example uses XNNPACK delegate by default. Users can change the
+[option in TfLiteInferenceCalculator](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/tflite/tflite_inference_calculator.proto)
+to use default TF Lite inference.
+
+## Graph
+
+### Main Graph
+
+![template_matching_mobile_graph](images/mobile/template_matching_mobile_graph.png)
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/template_matching/template_matching_mobile_cpu.pbtxt)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD

@@ -60,8 +60,8 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD

@@ -64,10 +64,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )