Internal change

PiperOrigin-RevId: 504341886

.github/ISSUE_TEMPLATE/11-tasks-issue.md

@@ -0,0 +1,25 @@
+---
+name: "Tasks Issue"
+about: Use this template for assistance with using MediaPipe Tasks (developers.google.com/mediapipe/solutions) to deploy on-device ML solutions (e.g. gesture recognition etc.) on supported platforms.
+labels: type:support
+
+---
+<em>Please make sure that this is a [Tasks](https://developers.google.com/mediapipe/solutions) issue.<em>
+
+**System information** (Please provide as much relevant information as possible)
+- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
+- MediaPipe Tasks SDK version:
+- Task name (e.g. Object detection, Gesture recognition etc.):
+- Programming Language and version ( e.g. C++, Python, Java):
+
+**Describe the expected behavior:**
+
+**Standalone code you may have used to try to get what you need :**
+
+If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
+
+**Other info / Complete Logs :**
+Include any logs or source code that would be helpful to
+diagnose the problem. If including tracebacks, please include the full
+traceback. Large logs and files should be attached:

.github/ISSUE_TEMPLATE/12-model-maker-issue.md

@@ -0,0 +1,25 @@
+---
+name: "Model Maker Issue"
+about: Use this template for assistance with using MediaPipe Model Maker (developers.google.com/mediapipe/solutions) to create custom on-device ML solutions.
+labels: type:support
+
+---
+<em>Please make sure that this is a [Model Maker](https://developers.google.com/mediapipe/solutions) issue.<em>
+
+**System information** (Please provide as much relevant information as possible)
+- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
+- Python version (e.g. 3.8):
+- [MediaPipe Model Maker version](https://pypi.org/project/mediapipe-model-maker/):
+- Task name (e.g. Image classification, Gesture recognition etc.):
+
+**Describe the expected behavior:**
+
+**Standalone code you may have used to try to get what you need :**
+
+If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
+
+**Other info / Complete Logs :**
+Include any logs or source code that would be helpful to
+diagnose the problem. If including tracebacks, please include the full
+traceback. Large logs and files should be attached:

.github/ISSUE_TEMPLATE/13-solution-issue.md

@@ -1,6 +1,6 @@
 ---
-name: "Solution Issue"
-about: Use this template for assistance with a specific mediapipe solution, such as "Pose" or "Iris", including inference model usage/training, solution-specific calculators, etc.
+name: "Solution (legacy) Issue"
+about: Use this template for assistance with a specific Mediapipe solution (google.github.io/mediapipe/solutions) such as "Pose", including inference model usage/training, solution-specific calculators etc.
 labels: type:support
 
 ---

.github/ISSUE_TEMPLATE/14-studio-issue.md

@@ -0,0 +1,19 @@
+---
+name: "Studio Issue"
+about: Use this template for assistance with the MediaPipe Studio application.
+labels: type:support
+
+---
+<em>Please make sure that this is a MediaPipe Studio issue.<em>
+
+**System information** (Please provide as much relevant information as possible)
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
+- Browser and Version
+- Any microphone or camera hardware
+- URL that shows the problem
+
+**Describe the expected behavior:**
+
+**Other info / Complete Logs :**
+Include any js console logs that would be helpful to diagnose the problem.
+Large logs and files should be attached:

.github/bot_config.yml

@@ -15,4 +15,5 @@
 
 # A list of assignees
 assignees:
-   - sureshdagooglecom
+   - kuaashish
+   - ayushgdev

.gitignore

@@ -2,5 +2,6 @@ bazel-*
 mediapipe/MediaPipe.xcodeproj
 mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user
 mediapipe/provisioning_profile.mobileprovision
+node_modules/
 .configure.bazelrc
 .user.bazelrc

BUILD.bazel

@@ -1,4 +1,4 @@
-# Copyright 2019 The MediaPipe Authors.
+# Copyright 2022 The MediaPipe Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,4 +14,9 @@
 
 licenses(["notice"])
 
-exports_files(["LICENSE"])
+exports_files([
+    "LICENSE",
+    "tsconfig.json",
+    "package.json",
+    "yarn.lock",
+])

Dockerfile

@@ -30,6 +30,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         wget \
         unzip \
+        nodejs \
+        npm \
         python3-dev \
         python3-opencv \
         python3-pip \
@@ -53,7 +55,7 @@ RUN pip3 install wheel
 RUN pip3 install future
 RUN pip3 install absl-py numpy opencv-contrib-python protobuf==3.20.1
 RUN pip3 install six==1.14.0
-RUN pip3 install tensorflow==2.2.0
+RUN pip3 install tensorflow
 RUN pip3 install tf_slim
 
 RUN ln -s /usr/bin/python3 /usr/bin/python

WORKSPACE

@@ -26,7 +26,7 @@ versions.check(minimum_bazel_version = "3.7.2")
 http_archive(
     name = "com_google_absl",
     urls = [
-        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz",
     ],
     # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
     patches = [
@@ -35,8 +35,8 @@ http_archive(
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "abseil-cpp-20210324.2",
-    sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f"
+    strip_prefix = "abseil-cpp-20220623.1",
+    sha256 = "91ac87d30cc6d79f9ab974c51874a704de9c2647c40f6932597329a282217ba8"
 )
 
 http_archive(
@@ -155,6 +155,14 @@ http_archive(
     name = "com_google_audio_tools",
     strip_prefix = "multichannel-audio-tools-master",
     urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
+    repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"},
+)
+
+http_archive(
+    name = "pffft",
+    strip_prefix = "jpommier-pffft-7c3b5a7dc510",
+    urls = ["https://bitbucket.org/jpommier/pffft/get/7c3b5a7dc510.zip"],
+    build_file = "@//third_party:pffft.BUILD",
 )
 
 # sentencepiece
@@ -165,7 +173,11 @@ http_archive(
     urls = [
         "https://github.com/google/sentencepiece/archive/1.0.0.zip",
     ],
-    repo_mapping = {"@com_google_glog" : "@com_github_glog_glog"},
+    patches = [
+        "//third_party:com_google_sentencepiece_no_gflag_no_gtest.diff",
+    ],
+    patch_args = ["-p1"],
+    repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
 )
 
 http_archive(
@@ -200,14 +212,14 @@ http_archive(
     sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91",
 )
 
-# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
+# 2022-10-20
 http_archive(
     name = "pybind11",
     urls = [
-        "https://github.com/pybind/pybind11/archive/70a58c577eaf067748c2ec31bfd0b0a614cffba6.zip",
+        "https://github.com/pybind/pybind11/archive/v2.10.1.zip",
     ],
-    sha256 = "b971842fab1b5b8f3815a2302331782b7d137fef0e06502422bc4bc360f4956c",
-    strip_prefix = "pybind11-70a58c577eaf067748c2ec31bfd0b0a614cffba6",
+    sha256 = "fcf94065efcfd0a7a828bacf118fa11c43f6390d0c805e3e6342ac119f2e9976",
+    strip_prefix = "pybind11-2.10.1",
     build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
 
@@ -308,12 +320,30 @@ http_archive(
     ],
 )
 
-# iOS basic build deps.
+# Load Zlib before initializing TensorFlow and the iOS build rules to guarantee
+# that the target @zlib//:mini_zlib is available
+http_archive(
+    name = "zlib",
+    build_file = "//third_party:zlib.BUILD",
+    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+    strip_prefix = "zlib-1.2.11",
+    urls = [
+        "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
+        "http://zlib.net/fossils/zlib-1.2.11.tar.gz",  # 2017-01-15
+    ],
+    patches = [
+        "@//third_party:zlib.diff",
+    ],
+    patch_args = [
+        "-p1",
+    ],
+)
 
+# iOS basic build deps.
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "77e8bf6fda706f420a55874ae6ee4df0c9d95da6c7838228b26910fc82eea5a2",
-    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.32.0/rules_apple.0.32.0.tar.gz",
+    sha256 = "f94e6dddf74739ef5cb30f000e13a2a613f6ebfa5e63588305a71fce8a8a9911",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/1.1.3/rules_apple.1.1.3.tar.gz",
     patches = [
         # Bypass checking ios unit test runner when building MP ios applications.
         "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
@@ -327,29 +357,24 @@ load(
     "@build_bazel_rules_apple//apple:repositories.bzl",
     "apple_rules_dependencies",
 )
-
 apple_rules_dependencies()
 
 load(
     "@build_bazel_rules_swift//swift:repositories.bzl",
     "swift_rules_dependencies",
 )
-
 swift_rules_dependencies()
 
-http_archive(
-    name = "build_bazel_apple_support",
-    sha256 = "741366f79d900c11e11d8efd6cc6c66a31bfb2451178b58e0b5edc6f1db17b35",
-    urls = [
-        "https://github.com/bazelbuild/apple_support/releases/download/0.10.0/apple_support.0.10.0.tar.gz"
-    ],
+load(
+    "@build_bazel_rules_swift//swift:extras.bzl",
+    "swift_rules_extra_dependencies",
 )
+swift_rules_extra_dependencies()
 
 load(
     "@build_bazel_apple_support//lib:repositories.bzl",
     "apple_support_dependencies",
 )
-
 apple_support_dependencies()
 
 # More iOS deps.
@@ -430,25 +455,6 @@ http_archive(
     ],
 )
 
-# Load Zlib before initializing TensorFlow to guarantee that the target
-# @zlib//:mini_zlib is available
-http_archive(
-    name = "zlib",
-    build_file = "//third_party:zlib.BUILD",
-    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
-    strip_prefix = "zlib-1.2.11",
-    urls = [
-        "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
-        "http://zlib.net/fossils/zlib-1.2.11.tar.gz",  # 2017-01-15
-    ],
-    patches = [
-        "@//third_party:zlib.diff",
-    ],
-    patch_args = [
-        "-p1",
-    ],
-)
-
 # TensorFlow repo should always go after the other external dependencies.
 # TF on 2022-08-10.
 _TENSORFLOW_GIT_COMMIT = "af1d5bc4fbb66d9e6cc1cf89503014a99233583b"
@@ -490,5 +496,51 @@ libedgetpu_dependencies()
 load("@coral_crosstool//:configure.bzl", "cc_crosstool")
 cc_crosstool(name = "crosstool")
 
+
+# Node dependencies
+http_archive(
+    name = "build_bazel_rules_nodejs",
+    sha256 = "5aae76dced38f784b58d9776e4ab12278bc156a9ed2b1d9fcd3e39921dc88fda",
+    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.7.1/rules_nodejs-5.7.1.tar.gz"],
+)
+
+load("@build_bazel_rules_nodejs//:repositories.bzl", "build_bazel_rules_nodejs_dependencies")
+build_bazel_rules_nodejs_dependencies()
+
+# fetches nodejs, npm, and yarn
+load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install")
+node_repositories()
+yarn_install(
+    name = "npm",
+    package_json = "//:package.json",
+    yarn_lock = "//:yarn.lock",
+)
+
+# Protobuf for Node dependencies
+http_archive(
+    name = "rules_proto_grpc",
+    sha256 = "bbe4db93499f5c9414926e46f9e35016999a4e9f6e3522482d3760dc61011070",
+    strip_prefix = "rules_proto_grpc-4.2.0",
+    urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/4.2.0.tar.gz"],
+)
+
+http_archive(
+    name = "com_google_protobuf_javascript",
+    sha256 = "35bca1729532b0a77280bf28ab5937438e3dcccd6b31a282d9ae84c896b6f6e3",
+    strip_prefix = "protobuf-javascript-3.21.2",
+    urls = ["https://github.com/protocolbuffers/protobuf-javascript/archive/refs/tags/v3.21.2.tar.gz"],
+)
+
+load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains", "rules_proto_grpc_repos")
+rules_proto_grpc_toolchains()
+rules_proto_grpc_repos()
+
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+rules_proto_dependencies()
+rules_proto_toolchains()
+
 load("//third_party:external_files.bzl", "external_files")
 external_files()
+
+load("//third_party:wasm_files.bzl", "wasm_files")
+wasm_files()

docs/BUILD

@@ -0,0 +1,33 @@
+# Placeholder for internal Python strict binary compatibility macro.
+
+py_binary(
+    name = "build_py_api_docs",
+    srcs = ["build_py_api_docs.py"],
+    deps = [
+        "//mediapipe",
+        "//third_party/py/absl:app",
+        "//third_party/py/absl/flags",
+        "//third_party/py/tensorflow_docs",
+        "//third_party/py/tensorflow_docs/api_generator:generate_lib",
+        "//third_party/py/tensorflow_docs/api_generator:public_api",
+    ],
+)
+
+py_binary(
+    name = "build_java_api_docs",
+    srcs = ["build_java_api_docs.py"],
+    data = [
+        "//third_party/android/sdk:api/26.txt",
+        "//third_party/java/doclava/current:doclava.jar",
+        "//third_party/java/jsilver:jsilver_jar",
+    ],
+    env = {
+        "DOCLAVA_JAR": "$(location //third_party/java/doclava/current:doclava.jar)",
+        "JSILVER_JAR": "$(location //third_party/java/jsilver:jsilver_jar)",
+    },
+    deps = [
+        "//third_party/py/absl:app",
+        "//third_party/py/absl/flags",
+        "//third_party/py/tensorflow_docs/api_generator/gen_java",
+    ],
+)

docs/build_java_api_docs.py

@@ -0,0 +1,63 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Generate Java reference docs for MediaPipe."""
+import pathlib
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.api_generator import gen_java
+
+_OUT_DIR = flags.DEFINE_string('output_dir', '/tmp/mp_java/',
+                               'Write docs here.')
+
+_SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api_docs/java',
+                                 'Path prefix in the _toc.yaml')
+
+_ = flags.DEFINE_string('code_url_prefix', None,
+                        '[UNUSED] The url prefix for links to code.')
+
+_ = flags.DEFINE_bool(
+    'search_hints', True,
+    '[UNUSED] Include metadata search hints in the generated files')
+
+_ANDROID_SDK = pathlib.Path('android/sdk/api/26.txt')
+
+
+def main(_) -> None:
+  # Default to using a relative path to find the Java source.
+  mp_root = pathlib.Path(__file__)
+  while (mp_root := mp_root.parent).name != 'mediapipe':
+    # Find the nearest `mediapipe` dir.
+    pass
+
+  # Find the root from which all packages are relative.
+  root = mp_root.parent
+
+  # Externally, parts of the repo are nested inside a mediapipe/ directory
+  # that does not exist internally. Support both.
+  if (mp_root / 'mediapipe').exists():
+    mp_root = mp_root / 'mediapipe'
+
+  gen_java.gen_java_docs(
+      package='com.google.mediapipe',
+      source_path=mp_root / 'tasks/java',
+      output_dir=pathlib.Path(_OUT_DIR.value),
+      site_path=pathlib.Path(_SITE_PATH.value),
+      federated_docs={'https://developer.android.com': root / _ANDROID_SDK})
+
+
+if __name__ == '__main__':
+  app.run(main)

docs/build_model_maker_api_docs.py

@@ -0,0 +1,81 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""MediaPipe Model Maker reference docs generation script.
+
+This script generates API reference docs for the `mediapipe` PIP package.
+
+$> pip install -U git+https://github.com/tensorflow/docs mediapipe-model-maker
+$> python build_model_maker_api_docs.py
+"""
+
+import os
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.api_generator import generate_lib
+
+try:
+  # mediapipe has not been set up to work with bazel yet, so catch & report.
+  import mediapipe_model_maker  # pytype: disable=import-error
+except ImportError as e:
+  raise ImportError('Please `pip install mediapipe-model-maker`.') from e
+
+
+PROJECT_SHORT_NAME = 'mediapipe_model_maker'
+PROJECT_FULL_NAME = 'MediaPipe Model Maker'
+
+_OUTPUT_DIR = flags.DEFINE_string(
+    'output_dir',
+    default='/tmp/generated_docs',
+    help='Where to write the resulting docs.')
+
+_URL_PREFIX = flags.DEFINE_string(
+    'code_url_prefix',
+    'https://github.com/google/mediapipe/tree/master/mediapipe/model_maker',
+    'The url prefix for links to code.')
+
+_SEARCH_HINTS = flags.DEFINE_bool(
+    'search_hints', True,
+    'Include metadata search hints in the generated files')
+
+_SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api_docs/python',
+                                 'Path prefix in the _toc.yaml')
+
+
+def gen_api_docs():
+  """Generates API docs for the mediapipe-model-maker package."""
+
+  doc_generator = generate_lib.DocGenerator(
+      root_title=PROJECT_FULL_NAME,
+      py_modules=[(PROJECT_SHORT_NAME, mediapipe_model_maker)],
+      base_dir=os.path.dirname(mediapipe_model_maker.__file__),
+      code_url_prefix=_URL_PREFIX.value,
+      search_hints=_SEARCH_HINTS.value,
+      site_path=_SITE_PATH.value,
+      callbacks=[],
+  )
+
+  doc_generator.build(_OUTPUT_DIR.value)
+
+  print('Docs output to:', _OUTPUT_DIR.value)
+
+
+def main(_):
+  gen_api_docs()
+
+
+if __name__ == '__main__':
+  app.run(main)

docs/build_py_api_docs.py

@@ -0,0 +1,83 @@
+# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""MediaPipe reference docs generation script.
+
+This script generates API reference docs for the `mediapipe` PIP package.
+
+$> pip install -U git+https://github.com/tensorflow/docs mediapipe
+$> python build_py_api_docs.py
+"""
+
+import os
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.api_generator import generate_lib
+
+try:
+  # mediapipe has not been set up to work with bazel yet, so catch & report.
+  import mediapipe as mp  # pytype: disable=import-error
+except ImportError as e:
+  raise ImportError('Please `pip install mediapipe`.') from e
+
+
+PROJECT_SHORT_NAME = 'mp'
+PROJECT_FULL_NAME = 'MediaPipe'
+
+_OUTPUT_DIR = flags.DEFINE_string(
+    'output_dir',
+    default='/tmp/generated_docs',
+    help='Where to write the resulting docs.')
+
+_URL_PREFIX = flags.DEFINE_string(
+    'code_url_prefix',
+    'https://github.com/google/mediapipe/blob/master/mediapipe',
+    'The url prefix for links to code.')
+
+_SEARCH_HINTS = flags.DEFINE_bool(
+    'search_hints', True,
+    'Include metadata search hints in the generated files')
+
+_SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api/solutions/python',
+                                 'Path prefix in the _toc.yaml')
+
+
+def gen_api_docs():
+  """Generates API docs for the mediapipe package."""
+  if hasattr(mp, 'solutions'):
+    del mp.solutions
+
+  doc_generator = generate_lib.DocGenerator(
+      root_title=PROJECT_FULL_NAME,
+      py_modules=[(PROJECT_SHORT_NAME, mp)],
+      base_dir=os.path.dirname(mp.__file__),
+      code_url_prefix=_URL_PREFIX.value,
+      search_hints=_SEARCH_HINTS.value,
+      site_path=_SITE_PATH.value,
+      callbacks=[],
+  )
+
+  doc_generator.build(_OUTPUT_DIR.value)
+
+  print('Docs output to:', _OUTPUT_DIR.value)
+
+
+def main(_):
+  gen_api_docs()
+
+
+if __name__ == '__main__':
+  app.run(main)

docs/framework_concepts/graphs.md

@@ -143,6 +143,98 @@ Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
     }
     ```
 
+## Graph Options
+
+It is possible to specify a "graph options" protobuf for a MediaPipe graph
+similar to the [`Calculator Options`](calculators.md#calculator-options)
+protobuf specified for a MediaPipe calculator. These "graph options" can be
+specified where a graph is invoked, and used to populate calculator options and
+subgraph options within the graph.
+
+In a CalculatorGraphConfig, graph options can be specified for a subgraph
+exactly like calculator options, as shown below:
+
+```
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "image"
+  output_stream: "throttled_image"
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions] {
+      max_in_flight: 1
+    }
+  }
+}
+
+node {
+  calculator: "FaceDetectionSubgraph"
+  input_stream: "IMAGE:throttled_image"
+  node_options: {
+    [type.googleapis.com/mediapipe.FaceDetectionOptions] {
+      tensor_width: 192
+      tensor_height: 192
+    }
+  }
+}
+```
+
+In a CalculatorGraphConfig, graph options can be accepted and used to populate
+calculator options, as shown below:
+
+```
+graph_options: {
+  [type.googleapis.com/mediapipe.FaceDetectionOptions] {}
+}
+
+node: {
+  calculator: "ImageToTensorCalculator"
+  input_stream: "IMAGE:multi_backend_image"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageToTensorCalculatorOptions] {
+        keep_aspect_ratio: true
+        border_mode: BORDER_ZERO
+    }
+  }
+  option_value: "output_tensor_width:options/tensor_width"
+  option_value: "output_tensor_height:options/tensor_height"
+}
+
+node {
+  calculator: "InferenceCalculator"
+  node_options: {
+    [type.googleapis.com/mediapipe.InferenceCalculatorOptions] {}
+  }
+  option_value: "delegate:options/delegate"
+  option_value: "model_path:options/model_path"
+}
+```
+
+In this example, the `FaceDetectionSubgraph` accepts graph option protobuf
+`FaceDetectionOptions`. The `FaceDetectionOptions` is used to define some field
+values in the calculator options `ImageToTensorCalculatorOptions` and some field
+values in the subgraph options `InferenceCalculatorOptions`. The field values
+are defined using the `option_value:` syntax.
+
+In the `CalculatorGraphConfig::Node` protobuf, the fields `node_options:` and
+`option_value:` together define the option values for a calculator such as
+`ImageToTensorCalculator`. The `node_options:` field defines a set of literal
+constant values using the text protobuf syntax. Each `option_value:` field
+defines the value for one protobuf field using information from the enclosing
+graph, specifically from field values of the graph options of the enclosing
+graph. In the example above, the `option_value:`
+`"output_tensor_width:options/tensor_width"` defines the field
+`ImageToTensorCalculatorOptions.output_tensor_width` using the value of
+`FaceDetectionOptions.tensor_width`.
+
+The syntax of `option_value:` is similar to the syntax of `input_stream:`. The
+syntax is `option_value: "LHS:RHS"`. The LHS identifies a calculator option
+field and the RHS identifies a graph option field. More specifically, the LHS
+and RHS each consists of a series of protobuf field names identifying nested
+protobuf messages and fields separated by '/'. This is known as the "ProtoPath"
+syntax. Nested messages that are referenced in the LHS or RHS must already be
+defined in the enclosing protobuf in order to be traversed using
+`option_value:`.
+
 ## Cycles
 
 <!-- TODO: add discussion of PreviousLoopbackCalculator -->

docs/getting_started/cpp.md

@@ -54,7 +54,7 @@ Note: This currently works only on Linux, and please first follow
 
     ```bash
     GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu \
-      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
+      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop_live_gpu.pbtxt
     ```
 
     This will open up your webcam as long as it is connected and on. Any errors

docs/getting_started/python.md

@@ -141,3 +141,4 @@ Nvidia Jetson and Raspberry Pi, please read
     ```bash
     (mp_env)mediapipe$ python3 setup.py bdist_wheel
     ```
+7. Exit from the MediaPipe repo directory and launch the Python interpreter.

docs/solutions/holistic.md

@@ -259,6 +259,7 @@ mp_holistic = mp.solutions.holistic
 
 # For static images:
 IMAGE_FILES = []
+BG_COLOR = (192, 192, 192) # gray
 with mp_holistic.Holistic(
     static_image_mode=True,
     model_complexity=2,

docs/solutions/models.md

@@ -94,8 +94,6 @@ one over the other.
 
 *   [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
 *   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
-*   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
-*   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
 
 ### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
 

docs/solutions/pose.md

@@ -217,7 +217,7 @@ A list of pose landmarks. Each landmark consists of the following:
 
 *Fig 5. Example of MediaPipe Pose real-world 3D coordinates.* |
 :-----------------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_world_landmarks.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/mobile/pose_world_landmarks.mp4" type="video/mp4"></video> |
 
 Another list of pose landmarks in world coordinates. Each landmark consists of
 the following:
@@ -238,7 +238,7 @@ for usage details.
 
 *Fig 6. Example of MediaPipe Pose segmentation mask.* |
 :---------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
 
 ### Python Solution API
 

docs/solutions/selfie_segmentation.md

@@ -22,7 +22,7 @@ nav_order: 7
 
 *Fig 1. Example of MediaPipe Selfie Segmentation.* |
 :------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/selfie_segmentation_web.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/selfie_segmentation_web.mp4" type="video/mp4"></video> |
 
 MediaPipe Selfie Segmentation segments the prominent humans in the scene. It can
 run in real-time on both smartphones and laptops. The intended use cases include

mediapipe/__init__.py

@@ -1,14 +1,13 @@
-"""Copyright 2019 - 2020 The MediaPipe Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+# Copyright 2019 - 2022 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

mediapipe/calculators/audio/BUILD

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "mfcc_mel_calculators_proto",
     srcs = ["mfcc_mel_calculators.proto"],
@@ -197,7 +197,6 @@ cc_library(
         ":spectrogram_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:matrix",
-        "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:core_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:logging",
@@ -277,6 +276,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_test_util",
+        "@com_google_audio_tools//audio/dsp/mfcc",
         "@eigen_archive//:eigen3",
     ],
 )
@@ -352,6 +352,8 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:validate_type",
         "//mediapipe/util:time_series_test_util",
+        "@com_google_audio_tools//audio/dsp:resampler",
+        "@com_google_audio_tools//audio/dsp:resampler_q",
         "@com_google_audio_tools//audio/dsp:signal_vector_util",
         "@eigen_archive//:eigen3",
     ],

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -280,6 +280,13 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
       audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
                                                     &window);
       break;
+    case SpectrogramCalculatorOptions::SQRT_HANN: {
+      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                 &window);
+      absl::c_transform(window, window.begin(),
+                        [](double x) { return std::sqrt(x); });
+      break;
+    }
   }
 
   // Propagate settings down to the actual Spectrogram object.

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -68,6 +68,7 @@ message SpectrogramCalculatorOptions {
     HANN = 0;
     HAMMING = 1;
     COSINE = 2;
+    SQRT_HANN = 4;
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/core/BUILD

@@ -13,16 +13,24 @@
 # limitations under the License.
 #
 
+load("@bazel_skylib//lib:selects.bzl", "selects")
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 licenses(["notice"])
 
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
+
+selects.config_setting_group(
+    name = "ios_or_disable_gpu",
+    match_any = [
+        "//mediapipe/gpu:disable_gpu",
+        "//mediapipe:ios",
+    ],
+)
 
 mediapipe_proto_library(
     name = "concatenate_vector_calculator_proto",
     srcs = ["concatenate_vector_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -32,7 +40,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "dequantize_byte_array_calculator_proto",
     srcs = ["dequantize_byte_array_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -42,7 +49,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_cloner_calculator_proto",
     srcs = ["packet_cloner_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -52,7 +58,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_resampler_calculator_proto",
     srcs = ["packet_resampler_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -62,7 +67,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_thinner_calculator_proto",
     srcs = ["packet_thinner_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -72,7 +76,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "split_vector_calculator_proto",
     srcs = ["split_vector_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -82,7 +85,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "quantize_float_vector_calculator_proto",
     srcs = ["quantize_float_vector_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -92,7 +94,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "sequence_shift_calculator_proto",
     srcs = ["sequence_shift_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -102,7 +103,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "gate_calculator_proto",
     srcs = ["gate_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -112,7 +112,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "constant_side_packet_calculator_proto",
     srcs = ["constant_side_packet_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -124,7 +123,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "clip_vector_size_calculator_proto",
     srcs = ["clip_vector_size_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -134,7 +132,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "flow_limiter_calculator_proto",
     srcs = ["flow_limiter_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -144,7 +141,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "graph_profile_calculator_proto",
     srcs = ["graph_profile_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -154,7 +150,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "get_vector_item_calculator_proto",
     srcs = ["get_vector_item_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -164,7 +159,6 @@ mediapipe_proto_library(
 cc_library(
     name = "add_header_calculator",
     srcs = ["add_header_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -193,7 +187,6 @@ cc_library(
     name = "begin_loop_calculator",
     srcs = ["begin_loop_calculator.cc"],
     hdrs = ["begin_loop_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
@@ -216,16 +209,15 @@ cc_library(
     name = "end_loop_calculator",
     srcs = ["end_loop_calculator.cc"],
     hdrs = ["end_loop_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:collection_item_id",
-        "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -258,7 +250,6 @@ cc_test(
 cc_library(
     name = "concatenate_vector_calculator_hdr",
     hdrs = ["concatenate_vector_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -284,7 +275,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework/api2:node",
@@ -311,7 +301,6 @@ cc_library(
 cc_library(
     name = "concatenate_detection_vector_calculator",
     srcs = ["concatenate_detection_vector_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -323,11 +312,11 @@ cc_library(
 cc_library(
     name = "concatenate_proto_list_calculator",
     srcs = ["concatenate_proto_list_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -340,10 +329,10 @@ cc_test(
     srcs = ["concatenate_proto_list_calculator_test.cc"],
     deps = [
         ":concatenate_proto_list_calculator",
-        ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
@@ -371,7 +360,6 @@ cc_library(
     name = "clip_vector_size_calculator",
     srcs = ["clip_vector_size_calculator.cc"],
     hdrs = ["clip_vector_size_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":clip_vector_size_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -387,7 +375,6 @@ cc_library(
 cc_library(
     name = "clip_detection_vector_size_calculator",
     srcs = ["clip_detection_vector_size_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":clip_vector_size_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -401,7 +388,6 @@ cc_test(
     srcs = ["clip_vector_size_calculator_test.cc"],
     deps = [
         ":clip_vector_size_calculator",
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:timestamp",
@@ -415,9 +401,6 @@ cc_test(
 cc_library(
     name = "counting_source_calculator",
     srcs = ["counting_source_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
@@ -430,9 +413,6 @@ cc_library(
 cc_library(
     name = "make_pair_calculator",
     srcs = ["make_pair_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -461,9 +441,6 @@ cc_test(
 cc_library(
     name = "matrix_multiply_calculator",
     srcs = ["matrix_multiply_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -477,9 +454,6 @@ cc_library(
 cc_library(
     name = "matrix_subtract_calculator",
     srcs = ["matrix_subtract_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -493,9 +467,6 @@ cc_library(
 cc_library(
     name = "mux_calculator",
     srcs = ["mux_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -508,9 +479,6 @@ cc_library(
 cc_library(
     name = "non_zero_calculator",
     srcs = ["non_zero_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -556,9 +524,6 @@ cc_test(
 cc_library(
     name = "packet_cloner_calculator",
     srcs = ["packet_cloner_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         ":packet_cloner_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -587,7 +552,6 @@ cc_test(
 cc_library(
     name = "packet_inner_join_calculator",
     srcs = ["packet_inner_join_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
@@ -611,9 +575,8 @@ cc_test(
 cc_library(
     name = "packet_thinner_calculator",
     srcs = ["packet_thinner_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        ":packet_thinner_calculator_cc_proto",
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:video_stream_header",
@@ -630,7 +593,7 @@ cc_test(
     srcs = ["packet_thinner_calculator_test.cc"],
     deps = [
         ":packet_thinner_calculator",
-        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        ":packet_thinner_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:video_stream_header",
@@ -643,9 +606,6 @@ cc_test(
 cc_library(
     name = "pass_through_calculator",
     srcs = ["pass_through_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:status",
@@ -656,9 +616,6 @@ cc_library(
 cc_library(
     name = "round_robin_demux_calculator",
     srcs = ["round_robin_demux_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -670,9 +627,6 @@ cc_library(
 cc_library(
     name = "immediate_mux_calculator",
     srcs = ["immediate_mux_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
@@ -684,7 +638,6 @@ cc_library(
 cc_library(
     name = "packet_presence_calculator",
     srcs = ["packet_presence_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
@@ -713,7 +666,6 @@ cc_test(
 cc_library(
     name = "previous_loopback_calculator",
     srcs = ["previous_loopback_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
@@ -729,7 +681,6 @@ cc_library(
 cc_library(
     name = "flow_limiter_calculator",
     srcs = ["flow_limiter_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":flow_limiter_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -746,7 +697,6 @@ cc_library(
 cc_library(
     name = "string_to_int_calculator",
     srcs = ["string_to_int_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:integral_types",
@@ -759,7 +709,6 @@ cc_library(
 cc_library(
     name = "default_side_packet_calculator",
     srcs = ["default_side_packet_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
@@ -771,7 +720,6 @@ cc_library(
 cc_library(
     name = "side_packet_to_stream_calculator",
     srcs = ["side_packet_to_stream_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:logging",
@@ -822,11 +770,8 @@ cc_library(
     name = "packet_resampler_calculator",
     srcs = ["packet_resampler_calculator.cc"],
     hdrs = ["packet_resampler_calculator.h"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        ":packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/deps:mathutil",
@@ -850,7 +795,7 @@ cc_test(
     ],
     deps = [
         ":packet_resampler_calculator",
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        ":packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:video_stream_header",
@@ -884,7 +829,6 @@ cc_test(
 cc_test(
     name = "matrix_multiply_calculator_test",
     srcs = ["matrix_multiply_calculator_test.cc"],
-    visibility = ["//visibility:private"],
     deps = [
         ":matrix_multiply_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -900,7 +844,6 @@ cc_test(
 cc_test(
     name = "matrix_subtract_calculator_test",
     srcs = ["matrix_subtract_calculator_test.cc"],
-    visibility = ["//visibility:private"],
     deps = [
         ":matrix_subtract_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -918,10 +861,10 @@ cc_test(
     name = "flow_limiter_calculator_test",
     srcs = ["flow_limiter_calculator_test.cc"],
     deps = [
+        ":counting_source_calculator",
         ":flow_limiter_calculator",
         ":flow_limiter_calculator_cc_proto",
-        "//mediapipe/calculators/core:counting_source_calculator",
-        "//mediapipe/calculators/core:pass_through_calculator",
+        ":pass_through_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:test_calculators",
@@ -934,6 +877,7 @@ cc_test(
         "//mediapipe/framework/tool:simulation_clock",
         "//mediapipe/framework/tool:simulation_clock_executor",
         "//mediapipe/framework/tool:sink",
+        "//mediapipe/util:packet_test_util",
         "@com_google_absl//absl/time",
     ],
 )
@@ -949,14 +893,13 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:ret_check",
@@ -965,8 +908,7 @@ cc_library(
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
     ] + select({
-        "//mediapipe/gpu:disable_gpu": [],
-        "//mediapipe:ios": [],
+        ":ios_or_disable_gpu": [],
         "//conditions:default": [
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
         ],
@@ -995,7 +937,6 @@ cc_test(
 cc_library(
     name = "split_proto_list_calculator",
     srcs = ["split_proto_list_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -1027,7 +968,6 @@ cc_test(
 cc_library(
     name = "dequantize_byte_array_calculator",
     srcs = ["dequantize_byte_array_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":dequantize_byte_array_calculator_cc_proto",
         "//mediapipe/framework:calculator_context",
@@ -1053,7 +993,6 @@ cc_test(
 cc_library(
     name = "quantize_float_vector_calculator",
     srcs = ["quantize_float_vector_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":quantize_float_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_context",
@@ -1079,7 +1018,6 @@ cc_test(
 cc_library(
     name = "sequence_shift_calculator",
     srcs = ["sequence_shift_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":sequence_shift_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -1104,7 +1042,6 @@ cc_test(
 cc_library(
     name = "gate_calculator",
     srcs = ["gate_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":gate_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -1130,7 +1067,6 @@ cc_test(
 cc_library(
     name = "matrix_to_vector_calculator",
     srcs = ["matrix_to_vector_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -1166,7 +1102,6 @@ cc_test(
 cc_library(
     name = "merge_calculator",
     srcs = ["merge_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -1192,7 +1127,6 @@ cc_test(
 cc_library(
     name = "stream_to_side_packet_calculator",
     srcs = ["stream_to_side_packet_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
@@ -1218,7 +1152,6 @@ cc_test(
 cc_library(
     name = "constant_side_packet_calculator",
     srcs = ["constant_side_packet_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":constant_side_packet_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -1248,7 +1181,6 @@ cc_test(
 cc_library(
     name = "graph_profile_calculator",
     srcs = ["graph_profile_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":graph_profile_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -1290,14 +1222,14 @@ cc_library(
     name = "get_vector_item_calculator",
     srcs = ["get_vector_item_calculator.cc"],
     hdrs = ["get_vector_item_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":get_vector_item_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:packet",
         "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
         "//mediapipe/framework/api2:port",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -1319,11 +1251,35 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "vector_indices_calculator",
+    srcs = ["vector_indices_calculator.cc"],
+    hdrs = ["vector_indices_calculator.h"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "vector_indices_calculator_test",
+    srcs = ["vector_indices_calculator_test.cc"],
+    deps = [
+        ":vector_indices_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+    ],
+)
+
 cc_library(
     name = "vector_size_calculator",
     srcs = ["vector_size_calculator.cc"],
     hdrs = ["vector_size_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -1337,9 +1293,6 @@ cc_library(
 cc_library(
     name = "packet_sequencer_calculator",
     srcs = ["packet_sequencer_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:contract",
@@ -1357,7 +1310,7 @@ cc_test(
     srcs = ["packet_sequencer_calculator_test.cc"],
     deps = [
         ":packet_sequencer_calculator",
-        "//mediapipe/calculators/core:pass_through_calculator",
+        ":pass_through_calculator",
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:subgraph",
@@ -1374,13 +1327,53 @@ cc_library(
     name = "merge_to_vector_calculator",
     srcs = ["merge_to_vector_calculator.cc"],
     hdrs = ["merge_to_vector_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image",
         "@com_google_absl//absl/status",
     ],
     alwayslink = 1,
 )
+
+mediapipe_proto_library(
+    name = "bypass_calculator_proto",
+    srcs = ["bypass_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "bypass_calculator",
+    srcs = ["bypass_calculator.cc"],
+    deps = [
+        ":bypass_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "bypass_calculator_test",
+    srcs = ["bypass_calculator_test.cc"],
+    deps = [
+        ":bypass_calculator",
+        ":pass_through_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:switch_container",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+    ],
+)

mediapipe/calculators/core/bypass_calculator.cc

@@ -0,0 +1,165 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/calculators/core/bypass_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using mediapipe::BypassCalculatorOptions;
+
+// Defines a "bypass" channel to use in place of a disabled feature subgraph.
+// By default, all inputs are discarded and all outputs are ignored.
+// Certain input streams can be passed to corresponding output streams
+// by specifying them in "pass_input_stream" and "pass_output_stream" options.
+// All output streams are updated with timestamp bounds indicating completed
+// output.
+//
+// Note that this calculator is designed for use as a contained_node in a
+// SwitchContainer.  For this reason, any input and output tags are accepted,
+// and stream semantics are specified through BypassCalculatorOptions.
+//
+// Example config:
+//     node {
+//       calculator: "BypassCalculator"
+//       input_stream: "APPEARANCES:appearances_post_facenet"
+//       input_stream: "VIDEO:video_frame"
+//       input_stream: "FEATURE_CONFIG:feature_config"
+//       input_stream: "ENABLE:gaze_enabled"
+//       output_stream: "APPEARANCES:analyzed_appearances"
+//       output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+//       node_options: {
+//         [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
+//           pass_input_stream: "APPEARANCES"
+//           pass_output_stream: "APPEARANCES"
+//         }
+//       }
+//     }
+//
+class BypassCalculator : public Node {
+ public:
+  static constexpr mediapipe::api2::Input<int>::Optional kNotNeeded{"N_N_"};
+  MEDIAPIPE_NODE_CONTRACT(kNotNeeded);
+  using IdMap = std::map<CollectionItemId, CollectionItemId>;
+
+  // Returns the map of passthrough input and output stream ids.
+  static absl::StatusOr<IdMap> GetPassMap(
+      const BypassCalculatorOptions& options, const tool::TagMap& input_map,
+      const tool::TagMap& output_map) {
+    IdMap result;
+    auto& input_streams = options.pass_input_stream();
+    auto& output_streams = options.pass_output_stream();
+    int size = std::min(input_streams.size(), output_streams.size());
+    for (int i = 0; i < size; ++i) {
+      std::pair<std::string, int> in_tag, out_tag;
+      MP_RETURN_IF_ERROR(tool::ParseTagIndex(options.pass_input_stream(i),
+                                             &in_tag.first, &in_tag.second));
+      MP_RETURN_IF_ERROR(tool::ParseTagIndex(options.pass_output_stream(i),
+                                             &out_tag.first, &out_tag.second));
+      auto input_id = input_map.GetId(in_tag.first, in_tag.second);
+      auto output_id = output_map.GetId(out_tag.first, out_tag.second);
+      result[input_id] = output_id;
+    }
+    return result;
+  }
+
+  // Identifies all specified streams as "Any" packet type.
+  // Identifies passthrough streams as "Same" packet type.
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    auto options = cc->Options<BypassCalculatorOptions>();
+    RET_CHECK_EQ(options.pass_input_stream().size(),
+                 options.pass_output_stream().size());
+    ASSIGN_OR_RETURN(
+        auto pass_streams,
+        GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
+    std::set<CollectionItemId> pass_out;
+    for (auto entry : pass_streams) {
+      pass_out.insert(entry.second);
+      cc->Inputs().Get(entry.first).SetAny();
+      cc->Outputs().Get(entry.second).SetSameAs(&cc->Inputs().Get(entry.first));
+    }
+    for (auto id = cc->Inputs().BeginId(); id != cc->Inputs().EndId(); ++id) {
+      if (pass_streams.count(id) == 0) {
+        cc->Inputs().Get(id).SetAny();
+      }
+    }
+    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
+      if (pass_out.count(id) == 0) {
+        cc->Outputs().Get(id).SetAny();
+      }
+    }
+    for (auto id = cc->InputSidePackets().BeginId();
+         id != cc->InputSidePackets().EndId(); ++id) {
+      cc->InputSidePackets().Get(id).SetAny();
+    }
+    return absl::OkStatus();
+  }
+
+  // Saves the map of passthrough input and output stream ids.
+  absl::Status Open(CalculatorContext* cc) override {
+    auto options = cc->Options<BypassCalculatorOptions>();
+    ASSIGN_OR_RETURN(pass_streams_, GetPassMap(options, *cc->Inputs().TagMap(),
+                                               *cc->Outputs().TagMap()));
+    return absl::OkStatus();
+  }
+
+  // Copies packets between passthrough input and output streams.
+  // Updates timestamp bounds on all output streams.
+  absl::Status Process(CalculatorContext* cc) override {
+    std::set<CollectionItemId> pass_out;
+    for (auto entry : pass_streams_) {
+      pass_out.insert(entry.second);
+      auto& packet = cc->Inputs().Get(entry.first).Value();
+      if (packet.Timestamp() == cc->InputTimestamp()) {
+        cc->Outputs().Get(entry.second).AddPacket(packet);
+      }
+    }
+    Timestamp bound = cc->InputTimestamp().NextAllowedInStream();
+    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
+      if (pass_out.count(id) == 0) {
+        cc->Outputs().Get(id).SetNextTimestampBound(
+            std::max(cc->Outputs().Get(id).NextTimestampBound(), bound));
+      }
+    }
+    return absl::OkStatus();
+  }
+
+  // Close all output streams.
+  absl::Status Close(CalculatorContext* cc) override {
+    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
+      cc->Outputs().Get(id).Close();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  IdMap pass_streams_;
+};
+
+MEDIAPIPE_REGISTER_NODE(BypassCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/bypass_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message BypassCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional BypassCalculatorOptions ext = 481259677;
+  }
+
+  // Names an input stream or streams to pass through, by "TAG:index".
+  repeated string pass_input_stream = 1;
+
+  // Names an output stream or streams to pass through, by "TAG:index".
+  repeated string pass_output_stream = 2;
+}

mediapipe/calculators/core/bypass_calculator_test.cc

@@ -0,0 +1,304 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/types/optional.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+// A graph with using a BypassCalculator to pass through and ignore
+// most of its inputs and outputs.
+constexpr char kTestGraphConfig1[] = R"pb(
+  type: "AppearancesPassThroughSubgraph"
+  input_stream: "APPEARANCES:appearances"
+  input_stream: "VIDEO:video_frame"
+  input_stream: "FEATURE_CONFIG:feature_config"
+  output_stream: "APPEARANCES:passthrough_appearances"
+  output_stream: "FEDERATED_GAZE_OUTPUT:passthrough_federated_gaze_output"
+
+  node {
+    calculator: "BypassCalculator"
+    input_stream: "PASS:appearances"
+    input_stream: "IGNORE:0:video_frame"
+    input_stream: "IGNORE:1:feature_config"
+    output_stream: "PASS:passthrough_appearances"
+    output_stream: "IGNORE:passthrough_federated_gaze_output"
+    node_options: {
+      [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
+        pass_input_stream: "PASS"
+        pass_output_stream: "PASS"
+      }
+    }
+  }
+)pb";
+
+// A graph with using AppearancesPassThroughSubgraph as a do-nothing channel
+// for input frames and appearances.
+constexpr char kTestGraphConfig2[] = R"pb(
+  input_stream: "VIDEO_FULL_RES:video_frame"
+  input_stream: "APPEARANCES:input_appearances"
+  input_stream: "FEATURE_CONFIG:feature_config"
+  input_stream: "GAZE_ENABLED:gaze_enabled"
+  output_stream: "APPEARANCES:analyzed_appearances"
+  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+
+  node {
+    calculator: "SwitchContainer"
+    input_stream: "VIDEO:video_frame"
+    input_stream: "APPEARANCES:input_appearances"
+    input_stream: "FEATURE_CONFIG:feature_config"
+    input_stream: "ENABLE:gaze_enabled"
+    output_stream: "APPEARANCES:analyzed_appearances"
+    output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+    options {
+      [mediapipe.SwitchContainerOptions.ext] {
+        async_selection: true
+        contained_node: { calculator: "AppearancesPassThroughSubgraph" }
+      }
+    }
+  }
+)pb";
+
+// A graph with using BypassCalculator as a do-nothing channel
+// for input frames and appearances.
+constexpr char kTestGraphConfig3[] = R"pb(
+  input_stream: "VIDEO_FULL_RES:video_frame"
+  input_stream: "APPEARANCES:input_appearances"
+  input_stream: "FEATURE_CONFIG:feature_config"
+  input_stream: "GAZE_ENABLED:gaze_enabled"
+  output_stream: "APPEARANCES:analyzed_appearances"
+  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+
+  node {
+    calculator: "SwitchContainer"
+    input_stream: "VIDEO:video_frame"
+    input_stream: "APPEARANCES:input_appearances"
+    input_stream: "FEATURE_CONFIG:feature_config"
+    input_stream: "ENABLE:gaze_enabled"
+    output_stream: "APPEARANCES:analyzed_appearances"
+    output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+    options {
+      [mediapipe.SwitchContainerOptions.ext] {
+        async_selection: true
+        contained_node: {
+          calculator: "BypassCalculator"
+          node_options: {
+            [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
+              pass_input_stream: "APPEARANCES"
+              pass_output_stream: "APPEARANCES"
+            }
+          }
+        }
+      }
+    }
+  }
+)pb";
+
+// A graph with using BypassCalculator as a disabled-gate
+// for input frames and appearances.
+constexpr char kTestGraphConfig4[] = R"pb(
+  input_stream: "VIDEO_FULL_RES:video_frame"
+  input_stream: "APPEARANCES:input_appearances"
+  input_stream: "FEATURE_CONFIG:feature_config"
+  input_stream: "GAZE_ENABLED:gaze_enabled"
+  output_stream: "APPEARANCES:analyzed_appearances"
+  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
+
+  node {
+    calculator: "SwitchContainer"
+    input_stream: "ENABLE:gaze_enabled"
+    input_stream: "VIDEO:video_frame"
+    input_stream: "APPEARANCES:input_appearances"
+    input_stream: "FEATURE_CONFIG:feature_config"
+    output_stream: "VIDEO:video_frame_out"
+    output_stream: "APPEARANCES:analyzed_appearances"
+    output_stream: "FEATURE_CONFIG:feature_config_out"
+    options {
+      [mediapipe.SwitchContainerOptions.ext] {
+        contained_node: { calculator: "BypassCalculator" }
+        contained_node: { calculator: "PassThroughCalculator" }
+      }
+    }
+  }
+)pb";
+
+// Reports packet timestamp and string contents, or "<empty>"".
+std::string DebugString(Packet p) {
+  return absl::StrCat(p.Timestamp().DebugString(), ":",
+                      p.IsEmpty() ? "<empty>" : p.Get<std::string>());
+}
+
+// Shows a bypass subgraph that passes through one stream.
+TEST(BypassCalculatorTest, SubgraphChannel) {
+  CalculatorGraphConfig config_1 =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig1);
+  CalculatorGraphConfig config_2 =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig2);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize({config_1, config_2}, {}));
+
+  std::vector<std::string> analyzed_appearances;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "analyzed_appearances",
+      [&](const Packet& p) {
+        analyzed_appearances.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  std::vector<std::string> federated_gaze_output;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "federated_gaze_output",
+      [&](const Packet& p) {
+        federated_gaze_output.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:a1"));
+  EXPECT_THAT(federated_gaze_output, testing::ElementsAre("200:<empty>"));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+// Shows a BypassCalculator that passes through one stream.
+TEST(BypassCalculatorTest, CalculatorChannel) {
+  CalculatorGraphConfig config_3 =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig3);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize({config_3}, {}));
+
+  std::vector<std::string> analyzed_appearances;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "analyzed_appearances",
+      [&](const Packet& p) {
+        analyzed_appearances.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  std::vector<std::string> federated_gaze_output;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "federated_gaze_output",
+      [&](const Packet& p) {
+        federated_gaze_output.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:a1"));
+  EXPECT_THAT(federated_gaze_output, testing::ElementsAre("200:<empty>"));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+// Shows a BypassCalculator that discards all inputs when ENABLED is false.
+TEST(BypassCalculatorTest, GatedChannel) {
+  CalculatorGraphConfig config_3 =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig4);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize({config_3}, {}));
+
+  std::vector<std::string> analyzed_appearances;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "analyzed_appearances",
+      [&](const Packet& p) {
+        analyzed_appearances.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  std::vector<std::string> video_frame;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "video_frame_out",
+      [&](const Packet& p) {
+        video_frame.push_back(DebugString(p));
+        return absl::OkStatus();
+      },
+      true));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Close the gate.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "gaze_enabled", MakePacket<bool>(false).At(Timestamp(200))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Send packets at timestamp 200.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Only timestamps arrive from the BypassCalculator.
+  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:<empty>"));
+  EXPECT_THAT(video_frame, testing::ElementsAre("200:<empty>"));
+
+  // Open the gate.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "gaze_enabled", MakePacket<bool>(true).At(Timestamp(300))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Send packets at timestamp 300.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_appearances", MakePacket<std::string>("a2").At(Timestamp(300))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "video_frame", MakePacket<std::string>("v2").At(Timestamp(300))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feature_config", MakePacket<std::string>("f2").At(Timestamp(300))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Packets arrive from the PassThroughCalculator.
+  EXPECT_THAT(analyzed_appearances,
+              testing::ElementsAre("200:<empty>", "300:a2"));
+  EXPECT_THAT(video_frame, testing::ElementsAre("200:<empty>", "300:v2"));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/core/concatenate_proto_list_calculator.cc

@@ -18,6 +18,7 @@
 #include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -111,6 +112,22 @@ class ConcatenateLandmarkListCalculator
 };
 MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkListCalculator);
 
+class ConcatenateClassificationListCalculator
+    : public ConcatenateListsCalculator<Classification, ClassificationList> {
+ protected:
+  int ListSize(const ClassificationList& list) const override {
+    return list.classification_size();
+  }
+  const Classification GetItem(const ClassificationList& list,
+                               int idx) const override {
+    return list.classification(idx);
+  }
+  Classification* AddItem(ClassificationList& list) const override {
+    return list.add_classification();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
+
 }  // namespace api2
 }  // namespace mediapipe
 

mediapipe/calculators/core/concatenate_proto_list_calculator_test.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -70,6 +71,16 @@ void AddInputLandmarkLists(
   }
 }
 
+void AddInputClassificationLists(
+    const std::vector<ClassificationList>& input_classifications_vec,
+    int64 timestamp, CalculatorRunner* runner) {
+  for (int i = 0; i < input_classifications_vec.size(); ++i) {
+    runner->MutableInputs()->Index(i).packets.push_back(
+        MakePacket<ClassificationList>(input_classifications_vec[i])
+            .At(Timestamp(timestamp)));
+  }
+}
+
 TEST(ConcatenateNormalizedLandmarkListCalculatorTest, EmptyVectorInputs) {
   CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
                           /*options_string=*/"", /*num_inputs=*/3,
@@ -181,4 +192,39 @@ TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneEmptyStreamNoOutput) {
   EXPECT_EQ(0, outputs.size());
 }
 
+TEST(ConcatenateClassificationListCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("ConcatenateClassificationListCalculator",
+                          /*options_string=*/
+                          "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
+                          "{only_emit_if_all_present: true}",
+                          /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  auto input_0 = ParseTextProtoOrDie<ClassificationList>(R"pb(
+    classification: { index: 0 score: 0.2 label: "test_0" }
+    classification: { index: 1 score: 0.3 label: "test_1" }
+    classification: { index: 2 score: 0.4 label: "test_2" }
+  )pb");
+  auto input_1 = ParseTextProtoOrDie<ClassificationList>(R"pb(
+    classification: { index: 3 score: 0.2 label: "test_3" }
+    classification: { index: 4 score: 0.3 label: "test_4" }
+  )pb");
+  std::vector<ClassificationList> inputs = {input_0, input_1};
+  AddInputClassificationLists(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  auto result = outputs[0].Get<ClassificationList>();
+  EXPECT_THAT(ParseTextProtoOrDie<ClassificationList>(R"pb(
+                classification: { index: 0 score: 0.2 label: "test_0" }
+                classification: { index: 1 score: 0.3 label: "test_1" }
+                classification: { index: 2 score: 0.4 label: "test_2" }
+                classification: { index: 3 score: 0.2 label: "test_3" }
+                classification: { index: 4 score: 0.3 label: "test_4" }
+              )pb"),
+              EqualsProto(result));
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
 #include "tensorflow/lite/interpreter.h"
@@ -40,6 +41,9 @@ REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);
 typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
 REGISTER_CALCULATOR(EndLoopBooleanCalculator);
 
+typedef EndLoopCalculator<std::vector<float>> EndLoopFloatCalculator;
+REGISTER_CALCULATOR(EndLoopFloatCalculator);
+
 typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
     EndLoopRenderDataCalculator;
 REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
@@ -55,4 +59,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
     EndLoopDetectionCalculator;
 REGISTER_CALCULATOR(EndLoopDetectionCalculator);
 
+typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator;
+REGISTER_CALCULATOR(EndLoopMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.h

@@ -50,7 +50,7 @@ namespace mediapipe {
 //   calculator:    "EndLoopWithOutputCalculator"
 //   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
 //   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
 // }
 template <typename IterableT>
 class EndLoopCalculator : public CalculatorBase {

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -18,7 +18,6 @@
 
 #include "mediapipe/calculators/core/flow_limiter_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/header_util.h"
 
@@ -68,7 +67,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
 // FlowLimiterCalculator provides limited support for multiple input streams.
 // The first input stream is treated as the main input stream and successive
 // input streams are treated as auxiliary input streams.  The auxiliary input
-// streams are limited to timestamps passed on the main input stream.
+// streams are limited to timestamps allowed by the "ALLOW" stream.
 //
 class FlowLimiterCalculator : public CalculatorBase {
  public:
@@ -100,64 +99,11 @@ class FlowLimiterCalculator : public CalculatorBase {
           cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>());
     }
     input_queues_.resize(cc->Inputs().NumEntries(""));
+    allowed_[Timestamp::Unset()] = true;
     RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
     return absl::OkStatus();
   }
 
-  // Returns true if an additional frame can be released for processing.
-  // The "ALLOW" output stream indicates this condition at each input frame.
-  bool ProcessingAllowed() {
-    return frames_in_flight_.size() < options_.max_in_flight();
-  }
-
-  // Outputs a packet indicating whether a frame was sent or dropped.
-  void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
-    if (cc->Outputs().HasTag(kAllowTag)) {
-      cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
-    }
-  }
-
-  // Sets the timestamp bound or closes an output stream.
-  void SetNextTimestampBound(Timestamp bound, OutputStream* stream) {
-    if (bound > Timestamp::Max()) {
-      stream->Close();
-    } else {
-      stream->SetNextTimestampBound(bound);
-    }
-  }
-
-  // Returns true if a certain timestamp is being processed.
-  bool IsInFlight(Timestamp timestamp) {
-    return std::find(frames_in_flight_.begin(), frames_in_flight_.end(),
-                     timestamp) != frames_in_flight_.end();
-  }
-
-  // Releases input packets up to the latest settled input timestamp.
-  void ProcessAuxiliaryInputs(CalculatorContext* cc) {
-    Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound();
-    for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) {
-      // Release settled frames from each input queue.
-      while (!input_queues_[i].empty() &&
-             input_queues_[i].front().Timestamp() < settled_bound) {
-        Packet packet = input_queues_[i].front();
-        input_queues_[i].pop_front();
-        if (IsInFlight(packet.Timestamp())) {
-          cc->Outputs().Get("", i).AddPacket(packet);
-        }
-      }
-
-      // Propagate each input timestamp bound.
-      if (!input_queues_[i].empty()) {
-        Timestamp bound = input_queues_[i].front().Timestamp();
-        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
-      } else {
-        Timestamp bound =
-            cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream();
-        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
-      }
-    }
-  }
-
   // Releases input packets allowed by the max_in_flight constraint.
   absl::Status Process(CalculatorContext* cc) final {
     options_ = tool::RetrieveOptions(options_, cc->Inputs());
@@ -224,13 +170,97 @@ class FlowLimiterCalculator : public CalculatorBase {
     }
 
     ProcessAuxiliaryInputs(cc);
+
+    // Discard old ALLOW ranges.
+    Timestamp input_bound = InputTimestampBound(cc);
+    auto first_range = std::prev(allowed_.upper_bound(input_bound));
+    allowed_.erase(allowed_.begin(), first_range);
     return absl::OkStatus();
   }
 
+  int LedgerSize() {
+    int result = frames_in_flight_.size() + allowed_.size();
+    for (const auto& queue : input_queues_) {
+      result += queue.size();
+    }
+    return result;
+  }
+
+ private:
+  // Returns true if an additional frame can be released for processing.
+  // The "ALLOW" output stream indicates this condition at each input frame.
+  bool ProcessingAllowed() {
+    return frames_in_flight_.size() < options_.max_in_flight();
+  }
+
+  // Outputs a packet indicating whether a frame was sent or dropped.
+  void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
+    if (cc->Outputs().HasTag(kAllowTag)) {
+      cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
+    }
+    allowed_[ts] = allow;
+  }
+
+  // Returns true if a timestamp falls within a range of allowed timestamps.
+  bool IsAllowed(Timestamp timestamp) {
+    auto it = allowed_.upper_bound(timestamp);
+    return std::prev(it)->second;
+  }
+
+  // Sets the timestamp bound or closes an output stream.
+  void SetNextTimestampBound(Timestamp bound, OutputStream* stream) {
+    if (bound > Timestamp::Max()) {
+      stream->Close();
+    } else {
+      stream->SetNextTimestampBound(bound);
+    }
+  }
+
+  // Returns the lowest unprocessed input Timestamp.
+  Timestamp InputTimestampBound(CalculatorContext* cc) {
+    Timestamp result = Timestamp::Done();
+    for (int i = 0; i < input_queues_.size(); ++i) {
+      auto& queue = input_queues_[i];
+      auto& stream = cc->Inputs().Get("", i);
+      Timestamp bound = queue.empty()
+                            ? stream.Value().Timestamp().NextAllowedInStream()
+                            : queue.front().Timestamp();
+      result = std::min(result, bound);
+    }
+    return result;
+  }
+
+  // Releases input packets up to the latest settled input timestamp.
+  void ProcessAuxiliaryInputs(CalculatorContext* cc) {
+    Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound();
+    for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) {
+      // Release settled frames from each input queue.
+      while (!input_queues_[i].empty() &&
+             input_queues_[i].front().Timestamp() < settled_bound) {
+        Packet packet = input_queues_[i].front();
+        input_queues_[i].pop_front();
+        if (IsAllowed(packet.Timestamp())) {
+          cc->Outputs().Get("", i).AddPacket(packet);
+        }
+      }
+
+      // Propagate each input timestamp bound.
+      if (!input_queues_[i].empty()) {
+        Timestamp bound = input_queues_[i].front().Timestamp();
+        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
+      } else {
+        Timestamp bound =
+            cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream();
+        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
+      }
+    }
+  }
+
  private:
   FlowLimiterCalculatorOptions options_;
   std::vector<std::deque<Packet>> input_queues_;
   std::deque<Timestamp> frames_in_flight_;
+  std::map<Timestamp, bool> allowed_;
 };
 REGISTER_CALCULATOR(FlowLimiterCalculator);
 

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -15,6 +15,7 @@
 #include <algorithm>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "absl/time/clock.h"
@@ -32,6 +33,7 @@
 #include "mediapipe/framework/tool/simulation_clock.h"
 #include "mediapipe/framework/tool/simulation_clock_executor.h"
 #include "mediapipe/framework/tool/sink.h"
+#include "mediapipe/util/packet_test_util.h"
 
 namespace mediapipe {
 
@@ -77,6 +79,12 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
   return result;
 }
 
+std::string SourceString(Timestamp t) {
+  return (t.IsSpecialValue())
+             ? t.DebugString()
+             : absl::StrCat("Timestamp(", t.DebugString(), ")");
+}
+
 // A Calculator::Process callback function.
 typedef std::function<absl::Status(const InputStreamShardSet&,
                                    OutputStreamShardSet*)>
@@ -651,11 +659,12 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
       input_packets_[17], input_packets_[19], input_packets_[20],
   };
   EXPECT_EQ(out_1_packets_, expected_output);
-  // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled.
+  // The timestamps released by FlowLimiterCalculator for in_1_sampled,
+  // plus input_packets_[21].
   std::vector<Packet> expected_output_2 = {
       input_packets_[0],  input_packets_[2],  input_packets_[4],
       input_packets_[14], input_packets_[17], input_packets_[19],
-      input_packets_[20],
+      input_packets_[20], input_packets_[21],
   };
   EXPECT_EQ(out_2_packets, expected_output_2);
 }
@@ -699,11 +708,10 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
         }
       )pb");
 
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
-    max_in_flight: 1
-    max_in_queue: 0
-    in_flight_timeout: 100000  # 100 ms
-  )pb");
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(
+      R"pb(
+        max_in_flight: 1 max_in_queue: 0 in_flight_timeout: 100000  # 100 ms
+      )pb");
   std::map<std::string, Packet> side_packets = {
       {"limiter_options",
        MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -759,13 +767,217 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
       input_packets_[0],  input_packets_[2],  input_packets_[15],
       input_packets_[17], input_packets_[19],
   };
-  EXPECT_EQ(out_1_packets_, expected_output);
+  EXPECT_THAT(out_1_packets_,
+              ElementsAreArray(PacketMatchers<int>(expected_output)));
+
   // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled.
   std::vector<Packet> expected_output_2 = {
       input_packets_[0],  input_packets_[2],  input_packets_[4],
       input_packets_[15], input_packets_[17], input_packets_[19],
   };
-  EXPECT_EQ(out_2_packets, expected_output_2);
+  EXPECT_THAT(out_2_packets,
+              ElementsAreArray(PacketMatchers<int>(expected_output_2)));
+
+  // Validate the ALLOW stream output.
+  std::vector<Packet> expected_allow = {
+      MakePacket<bool>(true).At(Timestamp(0)),
+      MakePacket<bool>(false).At(Timestamp(10000)),
+      MakePacket<bool>(true).At(Timestamp(20000)),
+      MakePacket<bool>(false).At(Timestamp(30000)),
+      MakePacket<bool>(true).At(Timestamp(40000)),
+      MakePacket<bool>(false).At(Timestamp(50000)),
+      MakePacket<bool>(false).At(Timestamp(60000)),
+      MakePacket<bool>(false).At(Timestamp(70000)),
+      MakePacket<bool>(false).At(Timestamp(80000)),
+      MakePacket<bool>(false).At(Timestamp(90000)),
+      MakePacket<bool>(false).At(Timestamp(100000)),
+      MakePacket<bool>(false).At(Timestamp(110000)),
+      MakePacket<bool>(false).At(Timestamp(120000)),
+      MakePacket<bool>(false).At(Timestamp(130000)),
+      MakePacket<bool>(false).At(Timestamp(140000)),
+      MakePacket<bool>(true).At(Timestamp(150000)),
+      MakePacket<bool>(false).At(Timestamp(160000)),
+      MakePacket<bool>(true).At(Timestamp(170000)),
+      MakePacket<bool>(false).At(Timestamp(180000)),
+      MakePacket<bool>(true).At(Timestamp(190000)),
+      MakePacket<bool>(false).At(Timestamp(200000)),
+  };
+  EXPECT_THAT(allow_packets_,
+              ElementsAreArray(PacketMatchers<bool>(expected_allow)));
+}
+
+std::vector<Packet> StripBoundsUpdates(const std::vector<Packet>& packets,
+                                       Timestamp begin = Timestamp::Min(),
+                                       Timestamp end = Timestamp::Max()) {
+  std::vector<Packet> result;
+  for (const auto& packet : packets) {
+    Timestamp ts = packet.Timestamp();
+    if (packet.IsEmpty() && ts >= begin && ts < end) {
+      continue;
+    }
+    result.push_back(packet);
+  }
+  return result;
+}
+
+// Shows how FlowLimiterCalculator releases auxiliary input packets.
+// In this test, auxiliary input packets arrive at twice the primary rate.
+TEST_F(FlowLimiterCalculatorTest, AuxiliaryInputs) {
+  // Configure the test.
+  SetUpInputData();
+  SetUpSimulationClock();
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input_1'
+        input_stream: 'auxiliary_input_2'
+        input_stream: 'auxiliary_input_3'
+        node {
+          calculator: 'FlowLimiterCalculator'
+          options {
+            [mediapipe.FlowLimiterCalculatorOptions.ext] {
+              max_in_flight: 1
+              max_in_queue: 0
+              in_flight_timeout: 1000000  # 1s
+            }
+          }
+          input_stream: 'input_1'
+          input_stream: 'auxiliary_input_2'
+          input_stream: 'auxiliary_input_3'
+          input_stream: 'FINISHED:out_1'
+          input_stream_info: { tag_index: 'FINISHED' back_edge: true }
+          output_stream: 'input_1_sampled'
+          output_stream: 'auxiliary_input_2_sampled'
+          output_stream: 'auxiliary_input_3_sampled'
+          output_stream: 'ALLOW:allow'
+        }
+        node {
+          calculator: 'SleepCalculator'
+          input_side_packet: 'WARMUP_TIME:warmup_time'
+          input_side_packet: 'SLEEP_TIME:sleep_time'
+          input_side_packet: 'CLOCK:clock'
+          input_stream: 'PACKET:input_1_sampled'
+          output_stream: 'PACKET:out_1'
+        }
+      )pb");
+
+  std::map<std::string, Packet> side_packets = {
+      // Fake processing lazy initialization time in microseconds.
+      {"warmup_time", MakePacket<int64>(22000)},
+      // Fake processing duration in microseconds.
+      {"sleep_time", MakePacket<int64>(22000)},
+      // The SimulationClock to count virtual elapsed time.
+      {"clock", MakePacket<mediapipe::Clock*>(clock_)},
+  };
+
+  // Start the graph.
+  MP_ASSERT_OK(graph_.Initialize(graph_config));
+  MP_EXPECT_OK(graph_.ObserveOutputStream(
+      "out_1",
+      [this](Packet p) {
+        out_1_packets_.push_back(p);
+        return absl::OkStatus();
+      },
+      true));
+  std::vector<Packet> out_2_packets, out_3_packets;
+  MP_EXPECT_OK(graph_.ObserveOutputStream(
+      "auxiliary_input_2_sampled",
+      [&](Packet p) {
+        out_2_packets.push_back(p);
+        return absl::OkStatus();
+      },
+      true));
+  MP_EXPECT_OK(graph_.ObserveOutputStream(
+      "auxiliary_input_3_sampled",
+      [&](Packet p) {
+        out_3_packets.push_back(p);
+        return absl::OkStatus();
+      },
+      true));
+  MP_EXPECT_OK(graph_.ObserveOutputStream(
+      "allow",
+      [this](Packet p) {
+        allow_packets_.push_back(p);
+        return absl::OkStatus();
+      },
+      true));
+  simulation_clock_->ThreadStart();
+  MP_ASSERT_OK(graph_.StartRun(side_packets));
+
+  // Add packets 1..9 to auxiliary_input_3, early.
+  for (int i = 1; i < 10; ++i) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        "auxiliary_input_3", MakePacket<int>(i).At(Timestamp(i * 10000))));
+  }
+
+  // The total count of out_2_packets after each input packet.
+  // std::vector<int> sizes_2 = {0, 0, 2, 2, 3, 3, 4, 4, 5, 5};
+  std::vector<int> sizes_2 = {0, 1, 3, 4, 6, 7, 9, 10, 12, 13};
+
+  // Add packets 2,4,6,8 to stream input_1.
+  // Add packets 1..9 to auxiliary_input_2.
+  for (int i = 1; i < 10; ++i) {
+    if (i % 2 == 0) {
+      MP_EXPECT_OK(graph_.AddPacketToInputStream(
+          "input_1", MakePacket<int>(i).At(Timestamp(i * 10000))));
+    }
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        "auxiliary_input_2", MakePacket<int>(i).At(Timestamp(i * 10000))));
+    clock_->Sleep(absl::Microseconds(10000));
+    EXPECT_EQ(out_2_packets.size(), sizes_2[i]);
+  }
+
+  // Finish the graph run.
+  MP_EXPECT_OK(graph_.CloseAllPacketSources());
+  clock_->Sleep(absl::Microseconds(40000));
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+  simulation_clock_->ThreadFinish();
+
+  // Validate the output.
+  // Input packets 4 and 8 are dropped due to max_in_flight.
+  std::vector<Packet> expected_output = {
+      MakePacket<int>(2).At(Timestamp(20000)),
+      Packet().At(Timestamp(40000)),
+      MakePacket<int>(6).At(Timestamp(60000)),
+      Packet().At(Timestamp(80000)),
+  };
+  EXPECT_THAT(out_1_packets_,
+              ElementsAreArray(PacketMatchers<int>(expected_output)));
+
+  // Packets following input packets 2 and 6, and not input packets 4 and 8.
+  std::vector<Packet> expected_auxiliary_output = {
+      Packet().At(Timestamp(9999)),
+      MakePacket<int>(1).At(Timestamp(10000)),
+      MakePacket<int>(2).At(Timestamp(20000)),
+      Packet().At(Timestamp(29999)),
+      MakePacket<int>(3).At(Timestamp(30000)),
+      Packet().At(Timestamp(40000)),
+      Packet().At(Timestamp(49999)),
+      Packet().At(Timestamp(50000)),
+      MakePacket<int>(6).At(Timestamp(60000)),
+      Packet().At(Timestamp(69999)),
+      MakePacket<int>(7).At(Timestamp(70000)),
+      Packet().At(Timestamp(80000)),
+      Packet().At(Timestamp(89999)),
+  };
+  std::vector<Packet> actual_2 =
+      StripBoundsUpdates(out_2_packets, Timestamp(90000));
+  EXPECT_THAT(actual_2,
+              ElementsAreArray(PacketMatchers<int>(expected_auxiliary_output)));
+  std::vector<Packet> expected_3 =
+      StripBoundsUpdates(expected_auxiliary_output, Timestamp(39999));
+  std::vector<Packet> actual_3 =
+      StripBoundsUpdates(out_3_packets, Timestamp(39999));
+  EXPECT_THAT(actual_3, ElementsAreArray(PacketMatchers<int>(expected_3)));
+
+  // Validate the ALLOW stream output.
+  std::vector<Packet> expected_allow = {
+      MakePacket<bool>(true).At(Timestamp(20000)),
+      MakePacket<bool>(false).At(Timestamp(40000)),
+      MakePacket<bool>(true).At(Timestamp(60000)),
+      MakePacket<bool>(false).At(Timestamp(80000)),
+  };
+  EXPECT_THAT(allow_packets_,
+              ElementsAreArray(PacketMatchers<bool>(expected_allow)));
 }
 
 }  // anonymous namespace

mediapipe/calculators/core/get_vector_item_calculator.cc

@@ -15,6 +15,7 @@
 #include "mediapipe/calculators/core/get_vector_item_calculator.h"
 
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 
 namespace mediapipe {
@@ -24,9 +25,17 @@ using GetLandmarkListVectorItemCalculator =
     GetVectorItemCalculator<mediapipe::LandmarkList>;
 REGISTER_CALCULATOR(GetLandmarkListVectorItemCalculator);
 
+using GetNormalizedLandmarkListVectorItemCalculator =
+    GetVectorItemCalculator<mediapipe::NormalizedLandmarkList>;
+REGISTER_CALCULATOR(GetNormalizedLandmarkListVectorItemCalculator);
+
 using GetClassificationListVectorItemCalculator =
     GetVectorItemCalculator<mediapipe::ClassificationList>;
 REGISTER_CALCULATOR(GetClassificationListVectorItemCalculator);
 
+using GetDetectionVectorItemCalculator =
+    GetVectorItemCalculator<mediapipe::Detection>;
+REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/get_vector_item_calculator.h

@@ -19,6 +19,7 @@
 
 #include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -46,7 +47,7 @@ namespace api2 {
 //     calculator: "Get{SpecificType}VectorItemCalculator"
 //     input_stream: "VECTOR:vector"
 //     input_stream: "INDEX:index"
-//     input_stream: "ITEM:item"
+//     output_stream: "ITEM:item"
 //     options {
 //       [mediapipe.GetVectorItemCalculatorOptions.ext] {
 //         item_index: 5
@@ -58,12 +59,13 @@ template <typename T>
 class GetVectorItemCalculator : public Node {
  public:
   static constexpr Input<std::vector<T>> kIn{"VECTOR"};
-  static constexpr Input<int>::Optional kIdx{"INDEX"};
+  static constexpr Input<OneOf<int, uint64_t>>::Optional kIdx{"INDEX"};
   static constexpr Output<T> kOut{"ITEM"};
 
   MEDIAPIPE_NODE_CONTRACT(kIn, kIdx, kOut);
 
   absl::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
     auto& options = cc->Options<mediapipe::GetVectorItemCalculatorOptions>();
     RET_CHECK(kIdx(cc).IsConnected() || options.has_item_index());
     return absl::OkStatus();
@@ -80,15 +82,21 @@ class GetVectorItemCalculator : public Node {
 
     int idx = 0;
     if (kIdx(cc).IsConnected() && !kIdx(cc).IsEmpty()) {
-      idx = kIdx(cc).Get();
+      idx = kIdx(cc).Visit(
+          [](uint64_t idx_uint64_t) { return static_cast<int>(idx_uint64_t); },
+          [](int idx_int) { return idx_int; });
     } else if (options.has_item_index()) {
       idx = options.item_index();
     } else {
       return absl::OkStatus();
     }
 
-    RET_CHECK(idx >= 0 && idx < items.size());
-    kOut(cc).Send(items[idx]);
+    RET_CHECK(idx >= 0);
+    RET_CHECK(options.output_empty_on_oob() || idx < items.size());
+
+    if (idx < items.size()) {
+      kOut(cc).Send(items[idx]);
+    }
 
     return absl::OkStatus();
   }

mediapipe/calculators/core/get_vector_item_calculator.proto

@@ -26,4 +26,7 @@ message GetVectorItemCalculatorOptions {
   // Index of vector item to get. INDEX input stream can be used instead, or to
   // override.
   optional int32 item_index = 1;
+
+  // Set to true to output an empty packet when the index is out of bounds.
+  optional bool output_empty_on_oob = 2;
 }

mediapipe/calculators/core/get_vector_item_calculator_test.cc

@@ -32,18 +32,21 @@ CalculatorRunner MakeRunnerWithStream() {
   )");
 }
 
-CalculatorRunner MakeRunnerWithOptions(int set_index) {
-  return CalculatorRunner(absl::StrFormat(R"(
+CalculatorRunner MakeRunnerWithOptions(int set_index,
+                                       bool output_empty_on_oob = false) {
+  return CalculatorRunner(
+      absl::StrFormat(R"(
     calculator: "TestGetIntVectorItemCalculator"
     input_stream: "VECTOR:vector_stream"
     output_stream: "ITEM:item_stream"
     options {
       [mediapipe.GetVectorItemCalculatorOptions.ext] {
         item_index: %d
+        output_empty_on_oob: %s
       }
     }
   )",
-                                          set_index));
+                      set_index, output_empty_on_oob ? "true" : "false"));
 }
 
 void AddInputVector(CalculatorRunner& runner, const std::vector<int>& inputs,
@@ -140,8 +143,7 @@ TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail1) {
 
   absl::Status status = runner.Run();
   ASSERT_FALSE(status.ok());
-  EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+  EXPECT_THAT(status.message(), testing::HasSubstr("idx >= 0"));
 }
 
 TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
@@ -155,7 +157,8 @@ TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
   absl::Status status = runner.Run();
   ASSERT_FALSE(status.ok());
   EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+              testing::HasSubstr(
+                  "options.output_empty_on_oob() || idx < items.size()"));
 }
 
 TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
@@ -167,8 +170,7 @@ TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
 
   absl::Status status = runner.Run();
   ASSERT_FALSE(status.ok());
-  EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+  EXPECT_THAT(status.message(), testing::HasSubstr("idx >= 0"));
 }
 
 TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
@@ -181,7 +183,21 @@ TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
   absl::Status status = runner.Run();
   ASSERT_FALSE(status.ok());
   EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+              testing::HasSubstr(
+                  "options.output_empty_on_oob() || idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail3) {
+  const int try_index = 3;
+  CalculatorRunner runner = MakeRunnerWithOptions(try_index, true);
+  const std::vector<int> inputs = {1, 2, 3};
+
+  AddInputVector(runner, inputs, 1);
+
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre());
 }
 
 TEST(TestGetIntVectorItemCalculatorTest, IndexStreamTwoTimestamps) {
@@ -227,4 +243,15 @@ TEST(TestGetIntVectorItemCalculatorTest, IndexOptionsTwoTimestamps) {
               testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
 }
 
+TEST(TestGetIntVectorItemCalculatorTest, IndexUint64) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+  const std::vector<int> inputs = {1, 2, 3};
+  const uint64_t index = 1;
+  AddInputVector(runner, inputs, 1);
+  AddInputIndex(runner, index, 1);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/merge_to_vector_calculator.cc

@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "mediapipe/calculators/core/merge_to_vector_calculator.h"
 
+#include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image.h"
 
 namespace mediapipe {
@@ -23,5 +24,13 @@ namespace api2 {
 typedef MergeToVectorCalculator<mediapipe::Image> MergeImagesToVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(MergeImagesToVectorCalculator);
 
+typedef MergeToVectorCalculator<mediapipe::GpuBuffer>
+    MergeGpuBuffersToVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(MergeGpuBuffersToVectorCalculator);
+
+typedef MergeToVectorCalculator<mediapipe::Detection>
+    MergeDetectionsToVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(MergeDetectionsToVectorCalculator);
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/merge_to_vector_calculator.h

@@ -42,11 +42,20 @@ class MergeToVectorCalculator : public Node {
     return absl::OkStatus();
   }
 
+  absl::Status Open(::mediapipe::CalculatorContext* cc) {
+    cc->SetOffset(::mediapipe::TimestampDiff(0));
+    return absl::OkStatus();
+  }
+
   absl::Status Process(CalculatorContext* cc) {
     const int input_num = kIn(cc).Count();
-    std::vector<T> output_vector(input_num);
-    std::transform(kIn(cc).begin(), kIn(cc).end(), output_vector.begin(),
-                   [](const auto& elem) -> T { return elem.Get(); });
+    std::vector<T> output_vector;
+    for (auto it = kIn(cc).begin(); it != kIn(cc).end(); it++) {
+      const auto& elem = *it;
+      if (!elem.IsEmpty()) {
+        output_vector.push_back(elem.Get());
+      }
+    }
     kOut(cc).Send(output_vector);
     return absl::OkStatus();
   }

mediapipe/calculators/core/mux_calculator.cc

@@ -41,6 +41,10 @@ class MuxCalculator : public Node {
                           StreamHandler("MuxInputStreamHandler"));
 
   absl::Status Process(CalculatorContext* cc) final {
+    if (kSelect(cc).IsStream() && kSelect(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+
     int select = *kSelect(cc);
     RET_CHECK(0 <= select && select < kIn(cc).Count());
     if (!kIn(cc)[select].IsEmpty()) {

mediapipe/calculators/core/mux_calculator_test.cc

@@ -398,6 +398,95 @@ TEST(MuxCalculatorTest, HandleTimestampBoundUpdates) {
   MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
+TEST(MuxCalculatorTest, HandlesCloseGracefully) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "select"
+            input_stream: "value_0"
+            input_stream: "value_1"
+            node {
+              calculator: "MuxCalculator"
+              input_stream: "SELECT:select"
+              input_stream: "INPUT:0:value_0"
+              input_stream: "INPUT:1:value_1"
+              output_stream: "OUTPUT:output"
+            }
+          )pb");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  // Observe packets.
+  std::vector<Packet> output_packets;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "output",
+      [&output_packets](const Packet& p) -> absl::Status {
+        output_packets.push_back(p);
+        return absl::OkStatus();
+      },
+      /*observe_timestamp_bounds=*/true));
+
+  // Start graph.
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Add single packet wait for completion and close.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "value_0", MakePacket<int>(0).At(Timestamp(1000))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  EXPECT_TRUE(output_packets.empty());
+}
+
+TEST(MuxCalculatorTest, HandlesCloseGracefullyWithDeafultInputStreamHandler) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            # This is required in order for EXPECT_DEATH to work everywhere
+            executor { name: "" type: "ApplicationThreadExecutor" }
+
+            input_stream: "select"
+            input_stream: "value_0"
+            input_stream: "value_1"
+            node {
+              calculator: "MuxCalculator"
+              input_stream: "SELECT:select"
+              input_stream: "INPUT:0:value_0"
+              input_stream: "INPUT:1:value_1"
+              output_stream: "OUTPUT:output"
+              input_stream_handler {
+                input_stream_handler: "DefaultInputStreamHandler"
+              }
+            }
+          )pb");
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  // Observe packets.
+  std::vector<Packet> output_packets;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "output",
+      [&output_packets](const Packet& p) -> absl::Status {
+        output_packets.push_back(p);
+        return absl::OkStatus();
+      },
+      /*observe_timestamp_bounds=*/true));
+
+  // Start graph.
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Add single packet wait for completion and close.
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "value_0", MakePacket<int>(0).At(Timestamp(1000))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ASSERT_EQ(output_packets.size(), 1);
+  EXPECT_TRUE(output_packets[0].IsEmpty());
+}
+
 }  // namespace
 
 }  // namespace mediapipe

mediapipe/calculators/core/real_time_flow_limiter_calculator.cc

@@ -76,7 +76,11 @@ constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
 //   }
 //   output_stream: "gated_frames"
 // }
-class RealTimeFlowLimiterCalculator : public CalculatorBase {
+//
+// Please use FlowLimiterCalculator, which replaces this calculator and
+// defines a few additional configuration options.
+class ABSL_DEPRECATED("Use FlowLimiterCalculator instead.")
+    RealTimeFlowLimiterCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
     int num_data_streams = cc->Inputs().NumEntries("");

mediapipe/calculators/core/sequence_shift_calculator.cc

@@ -66,12 +66,16 @@ class SequenceShiftCalculator : public Node {
   // The number of packets or timestamps we need to store to output packet[i] at
   // the timestamp of packet[i + packet_offset]; equal to abs(packet_offset).
   int cache_size_;
+  bool emit_empty_packets_before_first_packet_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(SequenceShiftCalculator);
 
 absl::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
   packet_offset_ = kOffset(cc).GetOr(
       cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset());
+  emit_empty_packets_before_first_packet_ =
+      cc->Options<mediapipe::SequenceShiftCalculatorOptions>()
+          .emit_empty_packets_before_first_packet();
   cache_size_ = abs(packet_offset_);
   // An offset of zero is a no-op, but someone might still request it.
   if (packet_offset_ == 0) {
@@ -96,6 +100,8 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
     // Ready to output oldest packet with current timestamp.
     kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
     packet_cache_.pop_front();
+  } else if (emit_empty_packets_before_first_packet_) {
+    LOG(FATAL) << "Not supported yet";
   }
   // Store current packet for later output.
   packet_cache_.push_back(kIn(cc).packet());

mediapipe/calculators/core/sequence_shift_calculator.proto

@@ -23,4 +23,8 @@ message SequenceShiftCalculatorOptions {
     optional SequenceShiftCalculatorOptions ext = 107633927;
   }
   optional int32 packet_offset = 1 [default = -1];
+
+  // Emits empty packets before the first delayed packet is emitted. Takes
+  // effect only when packet offset is set to positive.
+  optional bool emit_empty_packets_before_first_packet = 2 [default = false];
 }

mediapipe/calculators/core/vector_indices_calculator.cc

@@ -0,0 +1,33 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/vector_indices_calculator.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using IntVectorIndicesCalculator = VectorIndicesCalculator<int>;
+REGISTER_CALCULATOR(IntVectorIndicesCalculator);
+
+using Uint64tVectorIndicesCalculator = VectorIndicesCalculator<uint64_t>;
+REGISTER_CALCULATOR(Uint64tVectorIndicesCalculator);
+
+using NormalizedLandmarkListVectorIndicesCalculator =
+    VectorIndicesCalculator<mediapipe::NormalizedLandmarkList>;
+REGISTER_CALCULATOR(NormalizedLandmarkListVectorIndicesCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/vector_indices_calculator.h

@@ -0,0 +1,65 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_
+
+#include <optional>
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+// Calculator that takes a vector and constructs an index range vector based on
+// the size of the input vector.
+//
+// Inputs:
+//   VECTOR - std::vector<T>
+//     Vector whose range of indices to return.
+//
+// Outputs:
+//   INDICES - std::vector<int>
+//     Indices vector of the input vector.
+//
+// Example config:
+//  node {
+//    calculator: "{SpecificType}VectorIndicesCalculator"
+//    input_stream: "VECTOR:vector"
+//    output_stream: "INDICES:indices"
+//  }
+//
+template <typename T>
+class VectorIndicesCalculator : public Node {
+ public:
+  static constexpr Input<std::vector<T>> kVector{"VECTOR"};
+  static constexpr Output<std::vector<int>> kRange{"INDICES"};
+
+  MEDIAPIPE_NODE_CONTRACT(kVector, kRange);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    // Get the size of the input vector.
+    const int vector_size = kVector(cc).Get().size();
+    std::vector<int> out_idxs(vector_size);
+    std::iota(out_idxs.begin(), out_idxs.end(), 0);
+    kRange(cc).Send(out_idxs);
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_

mediapipe/calculators/core/vector_indices_calculator_test.cc

@@ -0,0 +1,87 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/vector_indices_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+using ::testing::TestParamInfo;
+using ::testing::TestWithParam;
+using ::testing::Values;
+
+template <typename T>
+void AddInputVector(CalculatorRunner& runner, const std::vector<T>& inputs,
+                    int timestamp) {
+  runner.MutableInputs()->Tag("VECTOR").packets.push_back(
+      MakePacket<std::vector<T>>(inputs).At(Timestamp(timestamp)));
+}
+
+template <typename T>
+struct TestParams {
+  const std::string test_name;
+  const std::vector<T> inputs;
+  const int timestamp;
+  const std::vector<int> expected_indices;
+};
+
+class IntVectorIndicesCalculatorTest
+    : public testing::TestWithParam<TestParams<int>> {};
+
+TEST_P(IntVectorIndicesCalculatorTest, Succeeds) {
+  CalculatorRunner runner = CalculatorRunner(R"(
+    calculator: "IntVectorIndicesCalculator"
+    input_stream: "VECTOR:vector_stream"
+    output_stream: "INDICES:indices_stream"
+  )");
+  const std::vector<int>& inputs = GetParam().inputs;
+  std::vector<int> expected_indices(inputs.size());
+  AddInputVector(runner, inputs, GetParam().timestamp);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("INDICES").packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_THAT(outputs[0].Get<std::vector<int>>(),
+              testing::ElementsAreArray(GetParam().expected_indices));
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    IntVectorIndicesCalculatorTest, IntVectorIndicesCalculatorTest,
+    Values(TestParams<int>{
+               /* test_name= */ "IntVectorIndices",
+               /* inputs= */ {1, 2, 3},
+               /* timestamp= */ 1,
+               /* expected_indices= */ {0, 1, 2},
+           },
+           TestParams<int>{
+               /* test_name= */ "EmptyVector",
+               /* inputs= */ {},
+               /* timestamp= */ 1,
+               /* expected_indices= */ {},
+           }),
+    [](const TestParamInfo<IntVectorIndicesCalculatorTest::ParamType>& info) {
+      return info.param.test_name;
+    });
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -16,12 +16,11 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 licenses(["notice"])
 
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 
 mediapipe_proto_library(
     name = "opencv_image_encoder_calculator_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -31,7 +30,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "scale_image_calculator_proto",
     srcs = ["scale_image_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -42,7 +40,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "set_alpha_calculator_proto",
     srcs = ["set_alpha_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -52,7 +49,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "image_cropping_calculator_proto",
     srcs = ["image_cropping_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -62,7 +58,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "bilateral_filter_calculator_proto",
     srcs = ["bilateral_filter_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -72,7 +67,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "recolor_calculator_proto",
     srcs = ["recolor_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -83,7 +77,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "segmentation_smoothing_calculator_proto",
     srcs = ["segmentation_smoothing_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -93,7 +86,6 @@ mediapipe_proto_library(
 cc_library(
     name = "color_convert_calculator",
     srcs = ["color_convert_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
@@ -112,7 +104,6 @@ cc_library(
 cc_library(
     name = "opencv_encoded_image_to_image_frame_calculator",
     srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":opencv_encoded_image_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -127,7 +118,6 @@ cc_library(
 cc_library(
     name = "opencv_image_encoder_calculator",
     srcs = ["opencv_image_encoder_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":opencv_image_encoder_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -142,7 +132,6 @@ cc_library(
 cc_library(
     name = "opencv_put_text_calculator",
     srcs = ["opencv_put_text_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
@@ -156,11 +145,10 @@ cc_library(
 cc_library(
     name = "set_alpha_calculator",
     srcs = ["set_alpha_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":set_alpha_calculator_cc_proto",
-        "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
@@ -183,11 +171,10 @@ cc_library(
 cc_library(
     name = "bilateral_filter_calculator",
     srcs = ["bilateral_filter_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":bilateral_filter_calculator_cc_proto",
-        "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework:calculator_options_cc_proto",
         "@com_google_absl//absl/strings",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
@@ -209,11 +196,16 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "rotation_mode_proto",
+    srcs = ["rotation_mode.proto"],
+)
+
 mediapipe_proto_library(
     name = "image_transformation_calculator_proto",
     srcs = ["image_transformation_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
+        ":rotation_mode_proto",
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/gpu:scale_mode_proto",
@@ -236,8 +228,8 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
+        ":rotation_mode_cc_proto",
         ":image_transformation_calculator_cc_proto",
         "//mediapipe/framework:packet",
         "//mediapipe/framework:timestamp",
@@ -279,13 +271,12 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":image_cropping_calculator_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
@@ -322,7 +313,6 @@ cc_test(
 cc_library(
     name = "luminance_calculator",
     srcs = ["luminance_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -336,7 +326,6 @@ cc_library(
 cc_library(
     name = "sobel_edges_calculator",
     srcs = ["sobel_edges_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -350,15 +339,14 @@ cc_library(
 cc_library(
     name = "recolor_calculator",
     srcs = ["recolor_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":recolor_calculator_cc_proto",
+        "//mediapipe/util:color_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/util:color_cc_proto",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
     ] + select({
@@ -377,9 +365,6 @@ cc_library(
     name = "scale_image_utils",
     srcs = ["scale_image_utils.cc"],
     hdrs = ["scale_image_utils.h"],
-    visibility = [
-        "//mediapipe:__subpackages__",
-    ],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:logging",
@@ -392,12 +377,9 @@ cc_library(
 cc_library(
     name = "scale_image_calculator",
     srcs = ["scale_image_calculator.cc"],
-    visibility = [
-        "//visibility:public",
-    ],
     deps = [
+        ":scale_image_calculator_cc_proto",
         ":scale_image_utils",
-        "//mediapipe/calculators/image:scale_image_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_format_cc_proto",
         "//mediapipe/framework/formats:image_frame",
@@ -421,7 +403,6 @@ cc_library(
 mediapipe_proto_library(
     name = "image_clone_calculator_proto",
     srcs = ["image_clone_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -431,7 +412,6 @@ mediapipe_proto_library(
 cc_library(
     name = "image_clone_calculator",
     srcs = ["image_clone_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":image_clone_calculator_cc_proto",
         "//mediapipe/framework/api2:node",
@@ -451,7 +431,6 @@ cc_library(
 cc_library(
     name = "image_properties_calculator",
     srcs = ["image_properties_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework:calculator_framework",
@@ -516,7 +495,6 @@ cc_test(
 mediapipe_proto_library(
     name = "mask_overlay_calculator_proto",
     srcs = ["mask_overlay_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -526,7 +504,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "opencv_encoded_image_to_image_frame_calculator_proto",
     srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -536,7 +513,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "feature_detector_calculator_proto",
     srcs = ["feature_detector_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -546,7 +522,6 @@ mediapipe_proto_library(
 cc_library(
     name = "mask_overlay_calculator",
     srcs = ["mask_overlay_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":mask_overlay_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -562,7 +537,6 @@ cc_library(
 cc_library(
     name = "feature_detector_calculator",
     srcs = ["feature_detector_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":feature_detector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -589,7 +563,6 @@ cc_library(
 cc_library(
     name = "image_file_properties_calculator",
     srcs = ["image_file_properties_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_file_properties_cc_proto",
@@ -619,11 +592,10 @@ cc_test(
 cc_library(
     name = "segmentation_smoothing_calculator",
     srcs = ["segmentation_smoothing_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         ":segmentation_smoothing_calculator_cc_proto",
-        "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image",
@@ -716,7 +688,6 @@ cc_library(
 mediapipe_proto_library(
     name = "warp_affine_calculator_proto",
     srcs = ["warp_affine_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -728,7 +699,6 @@ cc_library(
     name = "warp_affine_calculator",
     srcs = ["warp_affine_calculator.cc"],
     hdrs = ["warp_affine_calculator.h"],
-    visibility = ["//visibility:public"],
     deps = [
         ":affine_transformation",
         ":warp_affine_calculator_cc_proto",
@@ -777,8 +747,8 @@ cc_test(
     tags = ["desktop_only_test"],
     deps = [
         ":affine_transformation",
+        ":image_transformation_calculator",
         ":warp_affine_calculator",
-        "//mediapipe/calculators/image:image_transformation_calculator",
         "//mediapipe/calculators/tensor:image_to_tensor_converter",
         "//mediapipe/calculators/tensor:image_to_tensor_utils",
         "//mediapipe/calculators/util:from_image_calculator",
@@ -809,7 +779,6 @@ cc_test(
 cc_library(
     name = "yuv_to_image_calculator",
     srcs = ["yuv_to_image_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",

mediapipe/calculators/image/affine_transformation_runner_gl.cc

@@ -92,8 +92,8 @@ class GlTextureWarpAffineRunner
 
       constexpr GLchar kVertShader[] = R"(
             in vec4 position;
-            in mediump vec4 texture_coordinate;
-            out mediump vec2 sample_coordinate;
+            in highp vec4 texture_coordinate;
+            out highp vec2 sample_coordinate;
             uniform mat4 transform_matrix;
 
             void main() {
@@ -104,7 +104,7 @@ class GlTextureWarpAffineRunner
           )";
 
       constexpr GLchar kFragShader[] = R"(
-            DEFAULT_PRECISION(mediump, float)
+            DEFAULT_PRECISION(highp, float)
             in vec2 sample_coordinate;
             uniform sampler2D input_texture;
 

mediapipe/calculators/image/color_convert_calculator.cc

@@ -38,6 +38,7 @@ void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
 
 constexpr char kRgbaInTag[] = "RGBA_IN";
 constexpr char kRgbInTag[] = "RGB_IN";
+constexpr char kBgrInTag[] = "BGR_IN";
 constexpr char kBgraInTag[] = "BGRA_IN";
 constexpr char kGrayInTag[] = "GRAY_IN";
 constexpr char kRgbaOutTag[] = "RGBA_OUT";
@@ -57,6 +58,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 //   RGB  -> RGBA
 //   RGBA -> BGRA
 //   BGRA -> RGBA
+//   BGR  -> RGB
 //
 // This calculator only supports a single input stream and output stream at a
 // time. If more than one input stream or output stream is present, the
@@ -69,6 +71,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 //   RGB_IN:        The input video stream (ImageFrame, SRGB).
 //   BGRA_IN:       The input video stream (ImageFrame, SBGRA).
 //   GRAY_IN:       The input video stream (ImageFrame, GRAY8).
+//   BGR_IN:        The input video stream (ImageFrame, SBGR).
 //
 // Output streams:
 //   RGBA_OUT:      The output video stream (ImageFrame, SRGBA).
@@ -122,6 +125,10 @@ absl::Status ColorConvertCalculator::GetContract(CalculatorContract* cc) {
     cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>();
   }
 
+  if (cc->Inputs().HasTag(kBgrInTag)) {
+    cc->Inputs().Tag(kBgrInTag).Set<ImageFrame>();
+  }
+
   if (cc->Outputs().HasTag(kRgbOutTag)) {
     cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>();
   }
@@ -194,6 +201,11 @@ absl::Status ColorConvertCalculator::Process(CalculatorContext* cc) {
     return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA,
                             cv::COLOR_RGBA2BGRA, cc);
   }
+  // BGR -> RGB
+  if (cc->Inputs().HasTag(kBgrInTag) && cc->Outputs().HasTag(kRgbOutTag)) {
+    return ConvertAndOutput(kBgrInTag, kRgbOutTag, ImageFormat::SRGB,
+                            cv::COLOR_BGR2RGB, cc);
+  }
 
   return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
          << "Unsupported image format conversion.";

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -37,7 +37,8 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 namespace mediapipe {
 
 namespace {
-
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::Rect;
 #if !MEDIAPIPE_DISABLE_GPU
 
 #endif  // !MEDIAPIPE_DISABLE_GPU

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -195,11 +195,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
   auto cc = absl::make_unique<CalculatorContext>(
       calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
   auto& inputs = cc->Inputs();
-  mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
+  Rect rect = ParseTextProtoOrDie<Rect>(
       R"pb(
         width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
       )pb");
-  inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
+  inputs.Tag(kRectTag).Value() = MakePacket<Rect>(rect);
   RectSpec expectRect = {
       .width = 1,
       .height = 1,

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
+#include "mediapipe/calculators/image/rotation_mode.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"

mediapipe/calculators/image/image_transformation_calculator.proto

@@ -16,20 +16,10 @@ syntax = "proto2";
 
 package mediapipe;
 
+import "mediapipe/calculators/image/rotation_mode.proto";
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/gpu/scale_mode.proto";
 
-// Counterclockwise rotation.
-message RotationMode {
-  enum Mode {
-    UNKNOWN = 0;
-    ROTATION_0 = 1;
-    ROTATION_90 = 2;
-    ROTATION_180 = 3;
-    ROTATION_270 = 4;
-  }
-}
-
 message ImageTransformationCalculatorOptions {
   extend CalculatorOptions {
     optional ImageTransformationCalculatorOptions ext = 251952830;

mediapipe/calculators/image/rotation_mode.proto

@@ -0,0 +1,31 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+option java_package = "com.google.mediapipe.calculator.proto";
+option java_outer_classname = "RotationModeProto";
+
+// Counterclockwise rotation.
+message RotationMode {
+  enum Mode {
+    UNKNOWN = 0;
+    ROTATION_0 = 1;
+    ROTATION_90 = 2;
+    ROTATION_180 = 3;
+    ROTATION_270 = 4;
+  }
+}

mediapipe/calculators/image/scale_image_utils.cc

@@ -142,6 +142,9 @@ absl::Status FindOutputDimensions(int input_width,             //
                                       static_cast<double>(input_height));
     try_width = (try_width / 2) * 2;
     try_height = (try_height / 2) * 2;
+    // The output width/height should be greater than 0.
+    try_width = std::max(try_width, 1);
+    try_height = std::max(try_height, 1);
 
     if (target_height <= 0 || try_height <= target_height) {
       // The resulting height based on the target width and aspect ratio
@@ -160,6 +163,9 @@ absl::Status FindOutputDimensions(int input_width,             //
                                      static_cast<double>(input_width));
     try_width = (try_width / 2) * 2;
     try_height = (try_height / 2) * 2;
+    // The output width/height should be greater than 0.
+    try_width = std::max(try_width, 1);
+    try_height = std::max(try_height, 1);
 
     if (target_width <= 0 || try_width <= target_width) {
       // The resulting width based on the target width and aspect ratio

mediapipe/calculators/image/scale_image_utils_test.cc

@@ -124,6 +124,16 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
                                     &output_width, &output_height));
   EXPECT_EQ(151, output_width);
   EXPECT_EQ(101, output_height);
+  // Scale to height 1.
+  MP_ASSERT_OK(FindOutputDimensions(10000, 10, 100, 0, 0, true, 2,
+                                    &output_width, &output_height));
+  EXPECT_EQ(100, output_width);
+  EXPECT_EQ(1, output_height);
+  // Scale to width 1.
+  MP_ASSERT_OK(FindOutputDimensions(10, 10000, 0, 100, 0, true, 2,
+                                    &output_width, &output_height));
+  EXPECT_EQ(1, output_width);
+  EXPECT_EQ(100, output_height);
 }
 
 // Tests scaling without keeping the aspect ratio fixed.

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -46,6 +46,40 @@ constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
 constexpr int kNumChannelsRGBA = 4;
 
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+
+// Combines an RGB cv::Mat and a single-channel alpha cv::Mat of the same
+// dimensions into an RGBA cv::Mat. Alpha may be read as uint8 or as another
+// numeric type; in the latter case, it is upscaled to values between 0 and 255
+// from an assumed input range of [0, 1). RGB and RGBA Mat's must be uchar.
+template <typename AlphaType>
+absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat,
+                             cv::Mat& output_mat) {
+  RET_CHECK_EQ(input_mat.rows, alpha_mat.rows);
+  RET_CHECK_EQ(input_mat.cols, alpha_mat.cols);
+  RET_CHECK_EQ(input_mat.rows, output_mat.rows);
+  RET_CHECK_EQ(input_mat.cols, output_mat.cols);
+
+  for (int i = 0; i < output_mat.rows; ++i) {
+    const uchar* in_ptr = input_mat.ptr<uchar>(i);
+    const AlphaType* alpha_ptr = alpha_mat.ptr<AlphaType>(i);
+    uchar* out_ptr = output_mat.ptr<uchar>(i);
+    for (int j = 0; j < output_mat.cols; ++j) {
+      const int out_idx = j * kNumChannelsRGBA;
+      const int in_idx = j * input_mat.channels();
+      const int alpha_idx = j * alpha_mat.channels();
+      out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
+      out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
+      out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
+      if constexpr (std::is_same<AlphaType, uchar>::value) {
+        out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0];
+      } else {
+        const AlphaType alpha = alpha_ptr[alpha_idx + 0];
+        out_ptr[out_idx + 3] = static_cast<uchar>(round(alpha * 255.0f));
+      }
+    }
+  }
+  return absl::OkStatus();
+}
 }  // namespace
 
 // A calculator for setting the alpha channel of an RGBA image.
@@ -250,28 +284,22 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
 
   const bool has_alpha_mask = cc->Inputs().HasTag(kInputAlphaTag) &&
                               !cc->Inputs().Tag(kInputAlphaTag).IsEmpty();
-  const bool use_alpa_mask = alpha_value_ < 0 && has_alpha_mask;
+  const bool use_alpha_mask = alpha_value_ < 0 && has_alpha_mask;
 
   // Setup alpha image and Update image in CPU.
-  if (use_alpa_mask) {
+  if (use_alpha_mask) {
     const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get<ImageFrame>();
     cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask);
-    RET_CHECK_EQ(input_mat.rows, alpha_mat.rows);
-    RET_CHECK_EQ(input_mat.cols, alpha_mat.cols);
 
-    for (int i = 0; i < output_mat.rows; ++i) {
-      const uchar* in_ptr = input_mat.ptr<uchar>(i);
-      uchar* alpha_ptr = alpha_mat.ptr<uchar>(i);
-      uchar* out_ptr = output_mat.ptr<uchar>(i);
-      for (int j = 0; j < output_mat.cols; ++j) {
-        const int out_idx = j * kNumChannelsRGBA;
-        const int in_idx = j * input_mat.channels();
-        const int alpha_idx = j * alpha_mat.channels();
-        out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
-        out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
-        out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
-        out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0];  // channel 0 of mask
-      }
+    const bool alpha_is_float = alpha_mat.type() == CV_32FC1;
+    RET_CHECK(alpha_is_float || alpha_mat.type() == CV_8UC1);
+
+    if (alpha_is_float) {
+      MP_RETURN_IF_ERROR(
+          MergeRGBA8Image<float>(input_mat, alpha_mat, output_mat));
+    } else {
+      MP_RETURN_IF_ERROR(
+          MergeRGBA8Image<uchar>(input_mat, alpha_mat, output_mat));
     }
   } else {
     const uchar alpha_value = std::min(std::max(0.0f, alpha_value_), 255.0f);

mediapipe/calculators/internal/BUILD

@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-licenses(["notice"])
-
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 
+licenses(["notice"])
+
 package(default_visibility = ["//visibility:private"])
 
 proto_library(
     name = "callback_packet_calculator_proto",
     srcs = ["callback_packet_calculator.proto"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
@@ -29,14 +29,14 @@ mediapipe_cc_proto_library(
     name = "callback_packet_calculator_cc_proto",
     srcs = ["callback_packet_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
     deps = [":callback_packet_calculator_proto"],
 )
 
 cc_library(
     name = "callback_packet_calculator",
     srcs = ["callback_packet_calculator.cc"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
     deps = [
         ":callback_packet_calculator_cc_proto",
         "//mediapipe/framework:calculator_base",

mediapipe/calculators/tensor/BUILD

@@ -24,12 +24,13 @@ load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
 
 licenses(["notice"])
 
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 
 exports_files(
     glob(["testdata/image_to_tensor/*"]),
     visibility = [
         "//mediapipe/calculators/image:__subpackages__",
+        "//mediapipe/util:__subpackages__",
     ],
 )
 
@@ -43,9 +44,6 @@ selects.config_setting_group(
 mediapipe_proto_library(
     name = "audio_to_tensor_calculator_proto",
     srcs = ["audio_to_tensor_calculator.proto"],
-    visibility = [
-        "//mediapipe/framework:mediapipe_internal",
-    ],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -55,9 +53,6 @@ mediapipe_proto_library(
 cc_library(
     name = "audio_to_tensor_calculator",
     srcs = ["audio_to_tensor_calculator.cc"],
-    visibility = [
-        "//mediapipe/framework:mediapipe_internal",
-    ],
     deps = [
         ":audio_to_tensor_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -67,13 +62,16 @@ cc_library(
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:time_series_util",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_audio_tools//audio/dsp:resampler_q",
+        "@com_google_audio_tools//audio/dsp:window_functions",
         "@org_tensorflow//tensorflow/lite/c:common",
+        "@pffft",
     ],
     alwayslink = 1,
 )
@@ -83,6 +81,7 @@ cc_test(
     srcs = ["audio_to_tensor_calculator_test.cc"],
     deps = [
         ":audio_to_tensor_calculator",
+        ":audio_to_tensor_calculator_cc_proto",
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
@@ -97,10 +96,266 @@ cc_test(
     ],
 )
 
+mediapipe_proto_library(
+    name = "tensors_to_audio_calculator_proto",
+    srcs = ["tensors_to_audio_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "tensors_to_audio_calculator",
+    srcs = ["tensors_to_audio_calculator.cc"],
+    deps = [
+        ":tensors_to_audio_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/status",
+        "@com_google_audio_tools//audio/dsp:window_functions",
+        "@pffft",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "tensors_to_audio_calculator_test",
+    srcs = ["tensors_to_audio_calculator_test.cc"],
+    deps = [
+        ":audio_to_tensor_calculator",
+        ":audio_to_tensor_calculator_cc_proto",
+        ":tensors_to_audio_calculator",
+        ":tensors_to_audio_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_audio_tools//audio/dsp:window_functions",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "feedback_tensors_calculator_proto",
+    srcs = ["feedback_tensors_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "feedback_tensors_calculator",
+    srcs = ["feedback_tensors_calculator.cc"],
+    deps = [
+        ":feedback_tensors_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:tensor",
+        "@com_google_absl//absl/status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "feedback_tensors_calculator_test",
+    srcs = ["feedback_tensors_calculator_test.cc"],
+    deps = [
+        ":feedback_tensors_calculator",
+        ":feedback_tensors_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@org_tensorflow//tensorflow/lite/c:common",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "bert_preprocessor_calculator_proto",
+    srcs = ["bert_preprocessor_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "bert_preprocessor_calculator",
+    srcs = ["bert_preprocessor_calculator.cc"],
+    deps = [
+        ":bert_preprocessor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/tasks/cc/core:utils",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "//mediapipe/tasks/cc/text/tokenizers:tokenizer",
+        "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
+        "//mediapipe/tasks/metadata:metadata_schema_cc",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "bert_preprocessor_calculator_test",
+    srcs = ["bert_preprocessor_calculator_test.cc"],
+    data = ["//mediapipe/tasks/testdata/text:bert_text_classifier_models"],
+    linkopts = ["-ldl"],
+    deps = [
+        ":bert_preprocessor_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/tasks/cc/core:utils",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+        "@com_google_sentencepiece//src:sentencepiece_processor",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "regex_preprocessor_calculator_proto",
+    srcs = ["regex_preprocessor_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "regex_preprocessor_calculator",
+    srcs = ["regex_preprocessor_calculator.cc"],
+    deps = [
+        ":regex_preprocessor_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "//mediapipe/tasks/cc/text/tokenizers:regex_tokenizer",
+        "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
+        "//mediapipe/tasks/metadata:metadata_schema_cc",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "regex_preprocessor_calculator_test",
+    srcs = ["regex_preprocessor_calculator_test.cc"],
+    data = ["//mediapipe/tasks/testdata/text:text_classifier_models"],
+    linkopts = ["-ldl"],
+    deps = [
+        ":regex_preprocessor_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:sink",
+        "//mediapipe/tasks/cc/core:utils",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+        "@com_google_sentencepiece//src:sentencepiece_processor",
+    ],
+)
+
+cc_library(
+    name = "text_to_tensor_calculator",
+    srcs = ["text_to_tensor_calculator.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:tensor",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "text_to_tensor_calculator_test",
+    srcs = ["text_to_tensor_calculator_test.cc"],
+    deps = [
+        ":text_to_tensor_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_graph",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:options_map",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "universal_sentence_encoder_preprocessor_calculator",
+    srcs = ["universal_sentence_encoder_preprocessor_calculator.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/tasks/cc/core:utils",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "universal_sentence_encoder_preprocessor_calculator_test",
+    srcs = ["universal_sentence_encoder_preprocessor_calculator_test.cc"],
+    data = ["//mediapipe/tasks/testdata/text:universal_sentence_encoder_qa"],
+    deps = [
+        ":universal_sentence_encoder_preprocessor_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:options_map",
+        "//mediapipe/tasks/cc/core:utils",
+        "//mediapipe/tasks/cc/metadata:metadata_extractor",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 mediapipe_proto_library(
     name = "inference_calculator_proto",
     srcs = ["inference_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -119,16 +374,8 @@ cc_library(
     name = "inference_calculator_interface",
     srcs = ["inference_calculator.cc"],
     hdrs = ["inference_calculator.h"],
-    copts = select({
-        # TODO: fix tensor.h not to require this, if possible
-        "//mediapipe:apple": [
-            "-x objective-c++",
-            "-fobjc-arc",  # enable reference-counting
-        ],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
     deps = [
+        ":inference_calculator_cc_proto",
         ":inference_calculator_options_lib",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/api2:node",
@@ -151,7 +398,6 @@ cc_library(
     name = "inference_calculator_gl",
     srcs = ["inference_calculator_gl.cc"],
     tags = ["nomac"],  # config problem with cpuinfo via TF
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_cc_proto",
         ":inference_calculator_interface",
@@ -159,6 +405,7 @@ cc_library(
         "//mediapipe/gpu:gl_calculator_helper",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:str_format",
         "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
     ],
     alwayslink = 1,
@@ -168,7 +415,6 @@ cc_library(
     name = "inference_calculator_gl_advanced",
     srcs = ["inference_calculator_gl_advanced.cc"],
     tags = ["nomac"],
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_interface",
         "@com_google_absl//absl/memory",
@@ -199,15 +445,16 @@ cc_library(
         "-framework MetalKit",
     ],
     tags = ["ios"],
-    visibility = ["//visibility:public"],
     deps = [
         "inference_calculator_interface",
+        "//mediapipe/framework/formats:tensor",
         "//mediapipe/gpu:MPPMetalHelper",
         "//mediapipe/gpu:MPPMetalUtil",
         "//mediapipe/gpu:gpu_buffer",
         "//mediapipe/objc:mediapipe_framework_ios",
         "//mediapipe/util/tflite:config",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings:str_format",
         "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
         "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
         "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
@@ -219,16 +466,8 @@ cc_library(
 cc_library(
     name = "inference_runner",
     hdrs = ["inference_runner.h"],
-    copts = select({
-        # TODO: fix tensor.h not to require this, if possible
-        "//mediapipe:apple": [
-            "-x objective-c++",
-            "-fobjc-arc",  # enable reference-counting
-        ],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/framework/formats:tensor",
         "@com_google_absl//absl/status:statusor",
     ],
@@ -238,17 +477,9 @@ cc_library(
     name = "inference_interpreter_delegate_runner",
     srcs = ["inference_interpreter_delegate_runner.cc"],
     hdrs = ["inference_interpreter_delegate_runner.h"],
-    copts = select({
-        # TODO: fix tensor.h not to require this, if possible
-        "//mediapipe:apple": [
-            "-x objective-c++",
-            "-fobjc-arc",  # enable reference-counting
-        ],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_runner",
+        "//mediapipe/framework:mediapipe_profiling",
         "//mediapipe/framework/api2:packet",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:ret_check",
@@ -256,6 +487,8 @@ cc_library(
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@org_tensorflow//tensorflow/lite:framework_stable",
+        "@org_tensorflow//tensorflow/lite:string_util",
+        "@org_tensorflow//tensorflow/lite/c:c_api_types",
         "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
     ],
 )
@@ -265,15 +498,6 @@ cc_library(
     srcs = [
         "inference_calculator_cpu.cc",
     ],
-    copts = select({
-        # TODO: fix tensor.h not to require this, if possible
-        "//mediapipe:apple": [
-            "-x objective-c++",
-            "-fobjc-arc",  # enable reference-counting
-        ],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_interface",
         ":inference_calculator_utils",
@@ -312,15 +536,6 @@ cc_library(
     srcs = [
         "inference_calculator_xnnpack.cc",
     ],
-    copts = select({
-        # TODO: fix tensor.h not to require this, if possible
-        "//mediapipe:apple": [
-            "-x objective-c++",
-            "-fobjc-arc",  # enable reference-counting
-        ],
-        "//conditions:default": [],
-    }),
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_interface",
         ":inference_calculator_utils",
@@ -336,7 +551,6 @@ cc_library(
 
 cc_library(
     name = "inference_calculator_gl_if_compute_shader_available",
-    visibility = ["//visibility:public"],
     deps = selects.with_or({
         ":compute_shader_unavailable": [],
         "//conditions:default": [
@@ -346,9 +560,12 @@ cc_library(
     }),
 )
 
+# This target provides the InferenceCalculator and a default set of implementations tailored for the
+# current build platforms. More implementations can be added as separate dependencies to a client;
+# for clients that want a narrower set of implementations than the default should see the comment on
+# inference_calculator_interface.
 cc_library(
     name = "inference_calculator",
-    visibility = ["//visibility:public"],
     deps = [
         ":inference_calculator_interface",
         ":inference_calculator_cpu",
@@ -362,7 +579,6 @@ cc_library(
 mediapipe_proto_library(
     name = "tensor_converter_calculator_proto",
     srcs = ["tensor_converter_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -387,7 +603,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensor_converter_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -406,6 +621,7 @@ cc_library(
 
 cc_library(
     name = "tensor_converter_calculator_gpu_deps",
+    visibility = ["//visibility:private"],
     deps = select({
         "//mediapipe:android": [
             "//mediapipe/gpu:gl_calculator_helper",
@@ -450,7 +666,6 @@ cc_test(
 mediapipe_proto_library(
     name = "tensors_to_detections_calculator_proto",
     srcs = ["tensors_to_detections_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -475,19 +690,18 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensors_to_detections_calculator_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:span",
         "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
         "//mediapipe/framework/port:ret_check",
     ] + selects.with_or({
         ":compute_shader_unavailable": [],
@@ -498,6 +712,7 @@ cc_library(
 
 cc_library(
     name = "tensors_to_detections_calculator_gpu_deps",
+    visibility = ["//visibility:private"],
     deps = select({
         "//mediapipe:ios": [
             "//mediapipe/gpu:MPPMetalUtil",
@@ -513,7 +728,6 @@ cc_library(
 mediapipe_proto_library(
     name = "tensors_to_landmarks_calculator_proto",
     srcs = ["tensors_to_landmarks_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -530,7 +744,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensors_to_landmarks_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -545,7 +758,6 @@ cc_library(
 mediapipe_proto_library(
     name = "landmarks_to_tensor_calculator_proto",
     srcs = ["landmarks_to_tensor_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -563,7 +775,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":landmarks_to_tensor_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -596,7 +807,6 @@ cc_test(
 mediapipe_proto_library(
     name = "tensors_to_floats_calculator_proto",
     srcs = ["tensors_to_floats_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -613,7 +823,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensors_to_floats_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
@@ -651,7 +860,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensors_to_classification_calculator_cc_proto",
         "@com_google_absl//absl/container:node_hash_map",
@@ -682,7 +890,6 @@ cc_library(
 mediapipe_proto_library(
     name = "tensors_to_classification_calculator_proto",
     srcs = ["tensors_to_classification_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -720,7 +927,6 @@ cc_library(
         "//conditions:default": [],
     }),
     features = ["-layering_check"],  # allow depending on image_to_tensor_calculator_gpu_deps
-    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_calculator_cc_proto",
         ":image_to_tensor_converter",
@@ -749,6 +955,7 @@ cc_library(
 
 cc_library(
     name = "image_to_tensor_calculator_gpu_deps",
+    visibility = ["//visibility:private"],
     deps = selects.with_or({
         "//mediapipe:android": [
             ":image_to_tensor_converter_gl_buffer",
@@ -772,7 +979,6 @@ cc_library(
 mediapipe_proto_library(
     name = "image_to_tensor_calculator_proto",
     srcs = ["image_to_tensor_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -818,6 +1024,7 @@ cc_test(
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/util:image_test_utils",
         "@com_google_absl//absl/flags:flag",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -834,7 +1041,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_utils",
         "//mediapipe/framework/formats:image",
@@ -854,7 +1060,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_converter",
         ":image_to_tensor_utils",
@@ -874,6 +1079,7 @@ cc_library(
     name = "image_to_tensor_converter_gl_buffer",
     srcs = ["image_to_tensor_converter_gl_buffer.cc"],
     hdrs = ["image_to_tensor_converter_gl_buffer.h"],
+    visibility = ["//visibility:private"],
     deps = ["//mediapipe/framework:port"] + selects.with_or({
         "//mediapipe:apple": [],
         "//conditions:default": [
@@ -907,6 +1113,7 @@ cc_library(
     name = "image_to_tensor_converter_gl_texture",
     srcs = ["image_to_tensor_converter_gl_texture.cc"],
     hdrs = ["image_to_tensor_converter_gl_texture.h"],
+    visibility = ["//visibility:private"],
     deps = ["//mediapipe/framework:port"] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": [
@@ -931,6 +1138,7 @@ cc_library(
     name = "image_to_tensor_converter_gl_utils",
     srcs = ["image_to_tensor_converter_gl_utils.cc"],
     hdrs = ["image_to_tensor_converter_gl_utils.h"],
+    visibility = ["//visibility:private"],
     deps = ["//mediapipe/framework:port"] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": [
@@ -960,6 +1168,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:private"],
     deps = ["//mediapipe/framework:port"] + select({
         "//mediapipe:apple": [
             ":image_to_tensor_converter",
@@ -967,7 +1176,6 @@ cc_library(
             "//mediapipe/gpu:MPPMetalHelper",
             "@com_google_absl//absl/strings",
             "//mediapipe/framework:calculator_framework",
-            "//mediapipe/framework/formats:rect_cc_proto",
             "//mediapipe/framework/formats:tensor",
             "//mediapipe/framework/port:ret_check",
             "//mediapipe/framework/port:status",
@@ -985,22 +1193,41 @@ cc_library(
     name = "image_to_tensor_utils",
     srcs = ["image_to_tensor_utils.cc"],
     hdrs = ["image_to_tensor_utils.h"],
-    visibility = ["//visibility:public"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
     deps = [
+        ":image_to_tensor_calculator_cc_proto",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/types:optional",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:statusor",
-        "@com_google_absl//absl/types:optional",
-    ],
+        "//mediapipe/gpu:gpu_origin_cc_proto",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": ["//mediapipe/gpu:gpu_buffer"],
+    }),
 )
 
 cc_test(
     name = "image_to_tensor_utils_test",
     srcs = ["image_to_tensor_utils_test.cc"],
     deps = [
+        ":image_to_tensor_calculator_cc_proto",
         ":image_to_tensor_utils",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
     ],
 )
 
@@ -1015,7 +1242,6 @@ selects.config_setting_group(
 mediapipe_proto_library(
     name = "tensors_to_segmentation_calculator_proto",
     srcs = ["tensors_to_segmentation_calculator.proto"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -1033,7 +1259,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         ":tensors_to_segmentation_calculator_cc_proto",
         "@com_google_absl//absl/strings:str_format",
@@ -1046,9 +1271,9 @@ cc_library(
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
         "//mediapipe/util:resource_util",
         "@org_tensorflow//tensorflow/lite:framework",
-        "//mediapipe/gpu:gpu_origin_cc_proto",
         "//mediapipe/framework/port:statusor",
     ] + selects.with_or({
         "//mediapipe/gpu:disable_gpu": [],
@@ -1091,7 +1316,6 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -12,9 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
 #include <algorithm>
+#include <cmath>
 #include <cstring>
 #include <memory>
 #include <string>
@@ -26,6 +25,7 @@
 #include "absl/status/statusor.h"
 #include "absl/strings/str_format.h"
 #include "audio/dsp/resampler_q.h"
+#include "audio/dsp/window_functions.h"
 #include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/packet.h"
@@ -34,19 +34,61 @@
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/time_series_util.h"
+#include "pffft.h"
 
 namespace mediapipe {
 namespace api2 {
+namespace {
+
+using Options = ::mediapipe::AudioToTensorCalculatorOptions;
+using DftTensorFormat = Options::DftTensorFormat;
+using FlushMode = Options::FlushMode;
+
+std::vector<float> HannWindow(int window_size, bool sqrt_hann) {
+  std::vector<float> hann_window(window_size);
+  audio_dsp::HannWindow().GetPeriodicSamples(window_size, &hann_window);
+  if (sqrt_hann) {
+    absl::c_transform(hann_window, hann_window.begin(),
+                      [](double x) { return std::sqrt(x); });
+  }
+  return hann_window;
+}
+
+// PFFFT only supports transforms for inputs of length N of the form
+// N = (2^a)*(3^b)*(5^c) where b >=0 and c >= 0 and a >= 5 for the real FFT.
+bool IsValidFftSize(int size) {
+  if (size <= 0) {
+    return false;
+  }
+  constexpr int kFactors[] = {2, 3, 5};
+  int factorization[] = {0, 0, 0};
+  int n = static_cast<int>(size);
+  for (int i = 0; i < 3; ++i) {
+    while (n % kFactors[i] == 0) {
+      n = n / kFactors[i];
+      ++factorization[i];
+    }
+  }
+  return factorization[0] >= 5 && n == 1;
+}
+
+}  // namespace
 
 // Converts audio buffers into tensors, possibly with resampling, buffering
 // and framing, according to specified inputs and options. All input audio
 // buffers will be first resampled from the input sample rate to the target
 // sample rate if they are not equal. The resampled audio data (with the
 // buffered samples from the previous runs in the streaming mode) will be broken
-// into fixed-sized, possibly overlapping frames. Finally, all frames will be
-// converted to and outputted as MediaPipe Tensors. The last output tensor will
-// be zero-padding if the remaining samples are insufficient.
+// into fixed-sized, possibly overlapping frames. If the calculator is not asked
+// to perform fft (the fft_size is not set in the calculator options), all
+// frames will be converted to and outputted as MediaPipe Tensors. The last
+// output tensor will be zero-padding if the remaining samples are insufficient.
+// Otherwise, when the fft_size is set and valid, the calculator will perform
+// fft on the fixed-sized audio frames, the complex DFT results will be
+// converted to and outputted as 2D MediaPipe float Tensors where the first
+// rows are the DFT real parts and the second rows are the DFT imagery parts.
 //
 // This calculator assumes that the input timestamps refer to the first
 // sample in each Matrix. The output timestamps follow this same convention.
@@ -86,11 +128,15 @@ namespace api2 {
 // Outputs:
 //   TENSORS - std::vector<Tensor>
 //     Vector containing a single Tensor that represents a fix-sized audio
-//     frame.
+//     frame or the complex DFT results.
 //   TIMESTAMPS - std::vector<Timestamp> @Optional
 //     Vector containing the output timestamps emitted by the current Process()
 //     invocation. In the non-streaming mode, the vector contains all of the
 //     output timestamps for an input audio buffer.
+//   DC_AND_NYQUIST - std::pair<float, float> @Optional.
+//     A pair of dc component and nyquist component. Only can be connected when
+//     the calculator performs fft (the fft_size is set in the calculator
+//     options).
 //
 // Example:
 // node {
@@ -116,12 +162,14 @@ class AudioToTensorCalculator : public Node {
   // such as sample rate.
   static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
   static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
+  static constexpr Output<std::pair<float, float>>::Optional kDcAndNyquistOut{
+      "DC_AND_NYQUIST"};
   // A vector of the output timestamps emitted by the current Process()
   // invocation. The packet timestamp is the last emitted timestamp.
   static constexpr Output<std::vector<Timestamp>>::Optional kTimestampsOut{
       "TIMESTAMPS"};
   MEDIAPIPE_NODE_CONTRACT(kAudioIn, kAudioSampleRateIn, kTensorsOut,
-                          kTimestampsOut);
+                          kDcAndNyquistOut, kTimestampsOut);
 
   static absl::Status UpdateContract(CalculatorContract* cc);
   absl::Status Open(CalculatorContext* cc);
@@ -138,6 +186,11 @@ class AudioToTensorCalculator : public Node {
   int frame_step_;
   bool stream_mode_;
   bool check_inconsistent_timestamps_;
+  int padding_samples_before_;
+  int padding_samples_after_;
+  FlushMode flush_mode_;
+  DftTensorFormat dft_tensor_format_;
+
   Timestamp initial_timestamp_ = Timestamp::Unstarted();
   int64 cumulative_input_samples_ = 0;
   Timestamp next_output_timestamp_ = Timestamp::Unstarted();
@@ -151,22 +204,33 @@ class AudioToTensorCalculator : public Node {
   Matrix sample_buffer_;
   int processed_buffer_cols_ = 0;
 
+  // The internal state of the FFT library.
+  PFFFT_Setup* fft_state_ = nullptr;
+  int fft_size_ = 0;
+  std::vector<float> fft_window_;
+  std::vector<float, Eigen::aligned_allocator<float>> fft_input_buffer_;
+  // pffft requires memory to work with to avoid using the stack.
+  std::vector<float, Eigen::aligned_allocator<float>> fft_workplace_;
+  std::vector<float, Eigen::aligned_allocator<float>> fft_output_;
+
   absl::Status ProcessStreamingData(CalculatorContext* cc, const Matrix& input);
   absl::Status ProcessNonStreamingData(CalculatorContext* cc,
                                        const Matrix& input);
 
   absl::Status SetupStreamingResampler(double input_sample_rate_);
   void AppendToSampleBuffer(Matrix buffer_to_append);
+  void AppendZerosToSampleBuffer(int num_samples);
 
   absl::StatusOr<std::vector<Tensor>> ConvertToTensor(
-      const Matrix& frame_to_convert);
-  absl::Status OutputTensors(const Matrix& buffer, bool should_flush,
+      const Matrix& block, std::vector<int> tensor_dims);
+  absl::Status OutputTensor(const Matrix& block, Timestamp timestamp,
+                            CalculatorContext* cc);
+  absl::Status ProcessBuffer(const Matrix& buffer, bool should_flush,
                              CalculatorContext* cc);
 };
 
 absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
-  const auto& options =
-      cc->Options<mediapipe::AudioToTensorCalculatorOptions>();
+  const auto& options = cc->Options<Options>();
   if (!options.has_num_channels() || !options.has_num_samples() ||
       !options.has_target_sample_rate()) {
     return absl::InvalidArgumentError(
@@ -174,13 +238,21 @@ absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
         "`num_channels`, `num_samples`, and `target_sample_rate`.");
   }
   if (options.stream_mode()) {
-    // Explicitly disables tiemstamp offset to disallow the timestamp bound
+    // Explicitly disables timestamp offset to disallow the timestamp bound
     // from the input streams to be propagated to the output streams.
     // In the streaming mode, the output timestamp bound is based on
     // next_output_timestamp_, which can be smaller than the current input
     // timestamps.
     cc->SetTimestampOffset(TimestampDiff::Unset());
   }
+  if (options.padding_samples_before() < 0 ||
+      options.padding_samples_after() < 0) {
+    return absl::InvalidArgumentError("Negative zero padding unsupported");
+  }
+  if (options.flush_mode() != Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX &&
+      options.flush_mode() != Options::PROCEED_AS_USUAL) {
+    return absl::InvalidArgumentError("Unsupported flush mode");
+  }
   return absl::OkStatus();
 }
 
@@ -202,6 +274,10 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
     check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
     sample_buffer_.resize(num_channels_, Eigen::NoChange);
   }
+  padding_samples_before_ = options.padding_samples_before();
+  padding_samples_after_ = options.padding_samples_after();
+  dft_tensor_format_ = options.dft_tensor_format();
+  flush_mode_ = options.flush_mode();
 
   RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
             !kAudioIn(cc).Header().IsEmpty())
@@ -217,6 +293,25 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
       source_sample_rate_ = input_header.sample_rate();
     }
   }
+  AppendZerosToSampleBuffer(padding_samples_before_);
+  if (options.has_fft_size()) {
+    RET_CHECK(IsValidFftSize(options.fft_size()))
+        << "FFT size must be of the form fft_size = (2^a)*(3^b)*(5^c) where b "
+           ">=0 and c >= 0 and a >= 5, the requested fft size is "
+        << options.fft_size();
+    RET_CHECK_EQ(1, num_channels_)
+        << "Currently only support applying FFT on mono channel.";
+    fft_size_ = options.fft_size();
+    fft_state_ = pffft_new_setup(fft_size_, PFFFT_REAL);
+    fft_window_ = HannWindow(fft_size_, /* sqrt_hann = */ false);
+    fft_input_buffer_.resize(fft_size_);
+    fft_workplace_.resize(fft_size_);
+    fft_output_.resize(fft_size_);
+  } else {
+    RET_CHECK(!kDcAndNyquistOut(cc).IsConnected())
+        << "The DC_AND_NYQUIST output stream can only be connected when the "
+           "calculator outputs fft tensors";
+  }
   return absl::OkStatus();
 }
 
@@ -262,7 +357,12 @@ absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
     resampler_->Flush(&resampled_buffer);
     AppendToSampleBuffer(std::move(resampled_buffer));
   }
-  return OutputTensors(sample_buffer_, /*should_flush=*/true, cc);
+  AppendZerosToSampleBuffer(padding_samples_after_);
+  MP_RETURN_IF_ERROR(ProcessBuffer(sample_buffer_, /*should_flush=*/true, cc));
+  if (fft_state_) {
+    pffft_destroy_setup(fft_state_);
+  }
+  return absl::OkStatus();
 }
 
 absl::Status AudioToTensorCalculator::ProcessStreamingData(
@@ -303,7 +403,7 @@ absl::Status AudioToTensorCalculator::ProcessStreamingData(
     }
   }
 
-  MP_RETURN_IF_ERROR(OutputTensors(sample_buffer_, /*should_flush=*/false, cc));
+  MP_RETURN_IF_ERROR(ProcessBuffer(sample_buffer_, /*should_flush=*/false, cc));
   // Removes the processed samples from the global sample buffer.
   sample_buffer_ = Matrix(sample_buffer_.rightCols(sample_buffer_.cols() -
                                                    processed_buffer_cols_ - 1));
@@ -323,9 +423,9 @@ absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
         input_frame);
     Eigen::Map<const Matrix> matrix_mapping(resampled.data(), num_channels_,
                                             resampled.size() / num_channels_);
-    return OutputTensors(matrix_mapping, /*should_flush=*/true, cc);
+    return ProcessBuffer(matrix_mapping, /*should_flush=*/true, cc);
   }
-  return OutputTensors(input_frame, /*should_flush=*/true, cc);
+  return ProcessBuffer(input_frame, /*should_flush=*/true, cc);
 }
 
 absl::Status AudioToTensorCalculator::SetupStreamingResampler(
@@ -344,6 +444,16 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
   return absl::OkStatus();
 }
 
+void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
+  CHECK_GE(num_samples, 0);  // Ensured by `UpdateContract`.
+  if (num_samples == 0) {
+    return;
+  }
+  sample_buffer_.conservativeResize(Eigen::NoChange,
+                                    sample_buffer_.cols() + num_samples);
+  sample_buffer_.rightCols(num_samples).setZero();
+}
+
 void AudioToTensorCalculator::AppendToSampleBuffer(Matrix buffer_to_append) {
   sample_buffer_.conservativeResize(
       Eigen::NoChange, sample_buffer_.cols() + buffer_to_append.cols());
@@ -351,49 +461,118 @@ void AudioToTensorCalculator::AppendToSampleBuffer(Matrix buffer_to_append) {
 }
 
 absl::StatusOr<std::vector<Tensor>> AudioToTensorCalculator::ConvertToTensor(
-    const Matrix& frame_to_convert) {
-  Tensor tensor(Tensor::ElementType::kFloat32,
-                Tensor::Shape({num_channels_, num_samples_}));
+    const Matrix& block, std::vector<int> tensor_dims) {
+  Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape(tensor_dims));
   auto buffer_view = tensor.GetCpuWriteView();
-  if (frame_to_convert.size() < num_channels_ * num_samples_) {
+  int total_size = 1;
+  for (int dim : tensor_dims) {
+    total_size *= dim;
+  }
+  if (block.size() < total_size) {
     std::memset(buffer_view.buffer<float>(), 0, tensor.bytes());
   }
-  std::memcpy(buffer_view.buffer<float>(), frame_to_convert.data(),
-              frame_to_convert.size() * sizeof(float));
+  std::memcpy(buffer_view.buffer<float>(), block.data(),
+              block.size() * sizeof(float));
   std::vector<Tensor> tensor_vector;
   tensor_vector.push_back(std::move(tensor));
   return tensor_vector;
 }
 
-absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
+absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
+                                                   Timestamp timestamp,
+                                                   CalculatorContext* cc) {
+  std::vector<Tensor> output_tensor;
+  if (fft_state_) {
+    Eigen::VectorXf time_series_data =
+        Eigen::VectorXf::Map(block.data(), block.size());
+    //  Window on input audio prior to FFT.
+    std::transform(time_series_data.begin(), time_series_data.end(),
+                   fft_window_.begin(), fft_input_buffer_.begin(),
+                   std::multiplies<float>());
+    pffft_transform_ordered(fft_state_, fft_input_buffer_.data(),
+                            fft_output_.data(), fft_workplace_.data(),
+                            PFFFT_FORWARD);
+    if (kDcAndNyquistOut(cc).IsConnected()) {
+      kDcAndNyquistOut(cc).Send(std::make_pair(fft_output_[0], fft_output_[1]),
+                                timestamp);
+    }
+    switch (dft_tensor_format_) {
+      case Options::WITH_NYQUIST: {
+        Matrix fft_output_matrix =
+            Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
+        fft_output_matrix.conservativeResize(Eigen::NoChange, fft_size_);
+        // The last two elements are Nyquist component.
+        fft_output_matrix(fft_size_ - 2) = fft_output_[1];  // Nyquist real part
+        fft_output_matrix(fft_size_ - 1) = 0.0f;  // Nyquist imagery part
+        ASSIGN_OR_RETURN(output_tensor, ConvertToTensor(fft_output_matrix,
+                                                        {2, fft_size_ / 2}));
+        break;
+      }
+      case Options::WITH_DC_AND_NYQUIST: {
+        Matrix fft_output_matrix =
+            Eigen::Map<const Matrix>(fft_output_.data(), 1, fft_size_);
+        fft_output_matrix.conservativeResize(Eigen::NoChange, fft_size_ + 2);
+        fft_output_matrix(1) = 0.0f;  // DC imagery part.
+        // The last two elements are  Nyquist component.
+        fft_output_matrix(fft_size_) = fft_output_[1];  // Nyquist real part
+        fft_output_matrix(fft_size_ + 1) = 0.0f;        // Nyquist imagery part
+        ASSIGN_OR_RETURN(
+            output_tensor,
+            ConvertToTensor(fft_output_matrix, {2, (fft_size_ + 2) / 2}));
+        break;
+      }
+      case Options::WITHOUT_DC_AND_NYQUIST: {
+        Matrix fft_output_matrix =
+            Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
+        ASSIGN_OR_RETURN(
+            output_tensor,
+            ConvertToTensor(fft_output_matrix, {2, (fft_size_ - 2) / 2}));
+        break;
+      }
+      default:
+        return absl::InvalidArgumentError("Unsupported dft tensor format.");
+    }
+
+  } else {
+    ASSIGN_OR_RETURN(output_tensor,
+                     ConvertToTensor(block, {num_channels_, num_samples_}));
+  }
+  kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
+  return absl::OkStatus();
+}
+
+absl::Status AudioToTensorCalculator::ProcessBuffer(const Matrix& buffer,
                                                     bool should_flush,
                                                     CalculatorContext* cc) {
+  const bool should_flush_at_timestamp_max =
+      stream_mode_ && should_flush &&
+      flush_mode_ == Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX;
   int next_frame_first_col = 0;
   std::vector<Timestamp> timestamps;
-  while ((!stream_mode_ || !should_flush) &&
-         next_frame_first_col + num_samples_ <= buffer.cols()) {
-    ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
-                                             0, next_frame_first_col,
-                                             num_channels_, num_samples_)));
-    kTensorsOut(cc).Send(std::move(output_tensor), next_output_timestamp_);
-    timestamps.push_back(next_output_timestamp_);
-    next_output_timestamp_ += round(frame_step_ / target_sample_rate_ *
-                                    Timestamp::kTimestampUnitsPerSecond);
-    next_frame_first_col += frame_step_;
+  if (!should_flush_at_timestamp_max) {
+    while (next_frame_first_col + num_samples_ <= buffer.cols()) {
+      MP_RETURN_IF_ERROR(OutputTensor(
+          buffer.block(0, next_frame_first_col, num_channels_, num_samples_),
+          next_output_timestamp_, cc));
+      timestamps.push_back(next_output_timestamp_);
+      next_output_timestamp_ += round(frame_step_ / target_sample_rate_ *
+                                      Timestamp::kTimestampUnitsPerSecond);
+      next_frame_first_col += frame_step_;
+    }
   }
   if (should_flush && next_frame_first_col < buffer.cols()) {
-    ASSIGN_OR_RETURN(auto output_tensor,
-                     ConvertToTensor(buffer.block(
-                         0, next_frame_first_col, num_channels_,
-                         std::min(num_samples_,
-                                  (int)buffer.cols() - next_frame_first_col))));
     // In the streaming mode, the flush happens in Close() and a packet at
     // Timestamp::Max() will be emitted. In the non-streaming mode, each
     // Process() invocation will process the entire buffer completely.
-    Timestamp timestamp =
-        stream_mode_ ? Timestamp::Max() : next_output_timestamp_;
+    Timestamp timestamp = should_flush_at_timestamp_max
+                              ? Timestamp::Max()
+                              : next_output_timestamp_;
+    MP_RETURN_IF_ERROR(OutputTensor(
+        buffer.block(
+            0, next_frame_first_col, num_channels_,
+            std::min(num_samples_, (int)buffer.cols() - next_frame_first_col)),
+        timestamp, cc));
     timestamps.push_back(timestamp);
-    kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
   }
   if (kTimestampsOut(cc).IsConnected()) {
     Timestamp timestamp = timestamps.back();

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -44,4 +44,41 @@ message AudioToTensorCalculatorOptions {
   // Set to false to disable checks for jitter in timestamp values. Useful with
   // live audio input.
   optional bool check_inconsistent_timestamps = 6 [default = true];
+
+  // Size of the fft in number of bins. If set, the calculator outputs fft
+  // tensors.
+  optional int64 fft_size = 7;
+
+  // The amount of padding samples to add before the audio after resampling.
+  // Note that the timestamps shift. Currently, only zero padding is supported.
+  optional int64 padding_samples_before = 8;
+
+  // The amount of padding samples to add after the audio after resampling.
+  // Currently, only zero padding is supported.
+  optional int64 padding_samples_after = 9;
+
+  // Determines the "flushing" behavior in stream mode.
+  enum FlushMode {
+    // Unspecified (causes an error). Won't be used because of the default.
+    NONE = 0;
+    // Emit a packet with the entire remainder at `Timestamp::Max`.
+    ENTIRE_TAIL_AT_TIMESTAMP_MAX = 1;
+    // Continue emitting framed packets with relevant timestamps.
+    PROCEED_AS_USUAL = 2;
+  }
+
+  optional FlushMode flush_mode = 10 [default = ENTIRE_TAIL_AT_TIMESTAMP_MAX];
+
+  enum DftTensorFormat {
+    DFT_TENSOR_FORMAT_UNKNOWN = 0;
+    // The output dft tensor without dc and nyquist components.
+    WITHOUT_DC_AND_NYQUIST = 1;
+    // The output dft tensor contains the nyquist component as the last
+    // two values.
+    WITH_NYQUIST = 2;
+    // The output dft tensor contains the dc component as the first two values
+    // and the nyquist component as the last two values.
+    WITH_DC_AND_NYQUIST = 3;
+  }
+  optional DftTensorFormat dft_tensor_format = 11 [default = WITH_NYQUIST];
 }

mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc

@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <cmath>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "absl/strings/substitute.h"
 #include "audio/dsp/resampler_q.h"
+#include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -32,6 +32,14 @@
 namespace mediapipe {
 namespace {
 
+using ::testing::Not;
+using Options = ::mediapipe::AudioToTensorCalculatorOptions;
+using FlushMode = Options::FlushMode;
+
+int DivideRoundedUp(int dividend, int divisor) {
+  return (dividend + divisor - 1) / divisor;
+}
+
 std::unique_ptr<Matrix> CreateTestMatrix(int num_channels, int num_samples,
                                          int timestamp) {
   auto matrix = std::make_unique<Matrix>(num_channels, num_samples);
@@ -292,16 +300,17 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
     num_iterations_ = num_iterations;
   }
 
-  int GetExpectedNumOfSamples() {
-    Matrix* expected_matrix =
-        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
-    return expected_matrix->cols();
-  }
+  int GetExpectedNumOfSamples() { return output_sample_buffer_->cols(); }
 
   void Run(int num_samples, int num_overlapping_samples,
-           double resampling_factor) {
+           double resampling_factor, int padding_before = 0,
+           int padding_after = 0, bool expect_init_error = false) {
     double input_sample_rate = 10000;
     double target_sample_rate = input_sample_rate * resampling_factor;
+    FlushMode flush_mode = (padding_before != 0 || padding_after != 0)
+                               ? Options::PROCEED_AS_USUAL
+                               : Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX;
+
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
         absl::Substitute(R"(
         input_stream: "audio"
@@ -319,16 +328,25 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
               num_overlapping_samples: $1
               target_sample_rate: $2
               stream_mode:true
+              padding_samples_before: $3
+              padding_samples_after: $4
+              flush_mode: $5
             }
           }
         }
         )",
                          /*$0=*/num_samples, /*$1=*/num_overlapping_samples,
-                         /*$2=*/target_sample_rate));
+                         /*$2=*/target_sample_rate, /*$3=*/padding_before,
+                         /*$4=*/padding_after, /*$5=*/flush_mode));
     tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
 
     // Run the graph.
-    MP_ASSERT_OK(graph_.Initialize(graph_config));
+    const absl::Status init_status = graph_.Initialize(graph_config);
+    if (expect_init_error) {
+      EXPECT_THAT(init_status, Not(IsOk()));
+      return;
+    }
+    MP_ASSERT_OK(init_status);
     MP_ASSERT_OK(graph_.StartRun({}));
     for (int i = 0; i < num_iterations_; ++i) {
       Timestamp input_timestamp(Timestamp::kTimestampUnitsPerSecond * i);
@@ -345,8 +363,18 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
     }
     MP_ASSERT_OK(graph_.CloseAllInputStreams());
     MP_ASSERT_OK(graph_.WaitUntilIdle());
-    if (resampling_factor != 1) {
-      resampled_buffer_ = ResampleBuffer(*sample_buffer_, resampling_factor);
+    if (resampling_factor == 1) {
+      output_sample_buffer_ = std::make_unique<Matrix>(*sample_buffer_);
+    } else {
+      output_sample_buffer_ =
+          ResampleBuffer(*sample_buffer_, resampling_factor);
+    }
+    if (padding_before != 0 || padding_after != 0) {
+      Matrix padded = Matrix::Zero(
+          2, padding_before + output_sample_buffer_->cols() + padding_after);
+      padded.block(0, padding_before, 2, output_sample_buffer_->cols()) =
+          *output_sample_buffer_;
+      output_sample_buffer_->swap(padded);
     }
   }
 
@@ -372,15 +400,13 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
     auto buffer = output_tensor.GetCpuReadView().buffer<float>();
     int num_values = output_tensor.shape().num_elements();
     std::vector<float> output_floats(buffer, buffer + num_values);
-    Matrix* expected_matrix =
-        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
     for (int i = 0; i < num_values; ++i) {
-      if (i + sample_offset >= expected_matrix->size()) {
+      if (i + sample_offset >= output_sample_buffer_->size()) {
         EXPECT_FLOAT_EQ(output_floats[i], 0);
       } else {
         EXPECT_NEAR(output_floats[i],
-                    expected_matrix->coeff((i + sample_offset) % 2,
-                                           (i + sample_offset) / 2),
+                    output_sample_buffer_->coeff((i + sample_offset) % 2,
+                                                 (i + sample_offset) / 2),
                     0.001)
             << "i=" << i << ", sample_offset=" << sample_offset
             << ", packet index=" << index;
@@ -391,7 +417,8 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
 
   // Fully close graph at end, otherwise calculator+tensors are destroyed
   // after calling WaitUntilDone().
-  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
+  absl::Status TryCloseGraph() { return graph_.WaitUntilDone(); }
+  void CloseGraph() { MP_EXPECT_OK(TryCloseGraph()); }
 
  private:
   int input_buffer_num_samples_ = 10;
@@ -399,7 +426,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
   CalculatorGraph graph_;
   std::vector<Packet> tensors_packets_;
   std::unique_ptr<Matrix> sample_buffer_;
-  std::unique_ptr<Matrix> resampled_buffer_;
+  std::unique_ptr<Matrix> output_sample_buffer_;
 };
 
 TEST_F(AudioToTensorCalculatorStreamingModeTest,
@@ -408,7 +435,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest,
       /*resampling_factor=*/1.0f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/10,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 5),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 5),
       /*timestamp_interval=*/500,
       /*output_last_at_close=*/false);
   CloseGraph();
@@ -419,7 +446,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputRemainingInCloseMethod) {
       /*resampling_factor=*/1.0f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/12,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 6),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 6),
       /*timestamp_interval=*/600,
       /*output_last_at_close=*/true);
   CloseGraph();
@@ -431,7 +458,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputOverlappingFp32Tensors) {
       /*resampling_factor=*/1.0f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/16,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 8),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 8),
       /*timestamp_interval=*/800,
       /*output_last_at_close=*/true);
   CloseGraph();
@@ -443,7 +470,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, Downsampling) {
       /*resampling_factor=*/0.5f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/512,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 256),
       /*timestamp_interval=*/51200,
       /*output_last_at_close=*/true);
   CloseGraph();
@@ -455,7 +482,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, DownsamplingWithOverlapping) {
       /*resampling_factor=*/0.5f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/384,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
       /*timestamp_interval=*/38400,
       /*output_last_at_close=*/true);
   CloseGraph();
@@ -467,7 +494,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, Upsampling) {
       /*resampling_factor=*/2.0f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/512,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 256),
       /*timestamp_interval=*/12800,
       /*output_last_at_close=*/true);
   CloseGraph();
@@ -479,12 +506,33 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, UpsamplingWithOverlapping) {
       /*resampling_factor=*/2.0f);
   CheckTensorsOutputPackets(
       /*sample_offset=*/384,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
       /*timestamp_interval=*/9600,
       /*output_last_at_close=*/true);
   CloseGraph();
 }
 
+TEST_F(AudioToTensorCalculatorStreamingModeTest,
+       UpsamplingWithOverlappingAndPadding) {
+  SetInputBufferNumSamplesPerChannel(1024);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/2.0f, /*padding_before=*/13, /*padding_after=*/999);
+  CheckTensorsOutputPackets(
+      /*sample_offset=*/384,
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
+      /*timestamp_interval=*/9600,
+      /*output_last_at_close=*/false);
+  CloseGraph();
+}
+
+TEST_F(AudioToTensorCalculatorStreamingModeTest, NegativePaddingUnsupported) {
+  SetInputBufferNumSamplesPerChannel(1024);
+  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
+      /*resampling_factor=*/2.0f, /*padding_before=*/13, /*padding_after=*/-3,
+      /*expect_init_error=*/true);
+  EXPECT_THAT(TryCloseGraph(), Not(IsOk()));
+}
+
 TEST_F(AudioToTensorCalculatorStreamingModeTest,
        OnlyOutputInCloseIfNoSufficientSamples) {
   SetNumIterations(1);
@@ -498,5 +546,122 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest,
   CloseGraph();
 }
 
+class AudioToTensorCalculatorFftTest : public ::testing::Test {
+ protected:
+  // Creates an audio matrix containing a single sample of 1.0 at a specified
+  // offset.
+  std::unique_ptr<Matrix> CreateImpulseSignalData(int64 num_samples,
+                                                  int impulse_offset_idx) {
+    Matrix impulse = Matrix::Zero(1, num_samples);
+    impulse(0, impulse_offset_idx) = 1.0;
+    return std::make_unique<Matrix>(std::move(impulse));
+  }
+
+  void ConfigGraph(int num_channels, int num_samples,
+                   int num_overlapping_samples, double sample_rate,
+                   int fft_size) {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        absl::Substitute(R"(
+        input_stream: "audio"
+        input_stream: "sample_rate"
+        output_stream: "tensors"
+        output_stream: "dc_and_nyquist"
+        node {
+          calculator: "AudioToTensorCalculator"
+          input_stream: "AUDIO:audio"
+          input_stream: "SAMPLE_RATE:sample_rate"
+          output_stream: "TENSORS:tensors"
+          output_stream: "DC_AND_NYQUIST:dc_and_nyquist"
+          options {
+            [mediapipe.AudioToTensorCalculatorOptions.ext] {
+              num_channels: $0
+              num_samples: $1
+              num_overlapping_samples: $2
+              target_sample_rate: $3
+              fft_size: $4
+            }
+          }
+        }
+        )",
+                         /*$0=*/num_channels,
+                         /*$1=*/num_samples,
+                         /*$2=*/num_overlapping_samples,
+                         /*$3=*/sample_rate, /*$4=*/fft_size));
+    std::vector<Packet> tensors_packets;
+    tool::AddVectorSink("tensors", &graph_config_, &tensors_packets_);
+    std::vector<Packet> dc_and_nyquist_packets;
+    tool::AddVectorSink("dc_and_nyquist", &graph_config_,
+                        &dc_and_nyquist_packets_);
+  }
+
+  void RunGraph(std::unique_ptr<Matrix> input_data, double sample_rate) {
+    MP_ASSERT_OK(graph_.Initialize(graph_config_));
+    MP_ASSERT_OK(graph_.StartRun({}));
+    MP_ASSERT_OK(graph_.AddPacketToInputStream(
+        "sample_rate", MakePacket<double>(sample_rate).At(Timestamp(0))));
+    MP_ASSERT_OK(graph_.AddPacketToInputStream(
+        "audio", MakePacket<Matrix>(*input_data).At(Timestamp(0))));
+    MP_ASSERT_OK(graph_.CloseAllInputStreams());
+    MP_ASSERT_OK(graph_.WaitUntilIdle());
+    ASSERT_EQ(tensors_packets_.size(), dc_and_nyquist_packets_.size());
+  }
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
+
+  std::vector<Packet> tensors_packets_;
+  std::vector<Packet> dc_and_nyquist_packets_;
+  CalculatorGraphConfig graph_config_;
+  CalculatorGraph graph_;
+};
+
+TEST_F(AudioToTensorCalculatorFftTest, TestInvalidFftSize) {
+  ConfigGraph(1, 320, 160, 16000, 103);
+  MP_ASSERT_OK(graph_.Initialize(graph_config_));
+  MP_ASSERT_OK(graph_.StartRun({}));
+  auto status = graph_.WaitUntilIdle();
+  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
+  EXPECT_THAT(status.message(),
+              ::testing::HasSubstr("FFT size must be of the form"));
+}
+
+TEST_F(AudioToTensorCalculatorFftTest, TestInvalidNumChannels) {
+  ConfigGraph(3, 320, 160, 16000, 256);
+  MP_ASSERT_OK(graph_.Initialize(graph_config_));
+  MP_ASSERT_OK(graph_.StartRun({}));
+  auto status = graph_.WaitUntilIdle();
+  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
+  EXPECT_THAT(
+      status.message(),
+      ::testing::HasSubstr("only support applying FFT on mono channel"));
+}
+
+TEST_F(AudioToTensorCalculatorFftTest, TestImpulseSignal) {
+  constexpr double sample_rate = 16000;
+  ConfigGraph(1, 320, 160, sample_rate, 320);
+  RunGraph(CreateImpulseSignalData(320, 160), sample_rate);
+  for (int i = 0; i < tensors_packets_.size(); ++i) {
+    const auto& tensors = tensors_packets_[i].Get<std::vector<Tensor>>();
+    ASSERT_EQ(1, tensors.size());
+    const Tensor& output_tensor =
+        tensors_packets_[0].Get<std::vector<Tensor>>()[0];
+    auto* buffer = output_tensor.GetCpuReadView().buffer<float>();
+    int num_values = output_tensor.shape().num_elements();
+    const std::vector<float> output_floats(buffer, buffer + num_values);
+    // Impulse signal should have (approximately) const power across all
+    // frequency bins.
+    const auto& pair =
+        dc_and_nyquist_packets_[i].Get<std::pair<float, float>>();
+    EXPECT_FLOAT_EQ(pair.first, 1.0f);
+    EXPECT_FLOAT_EQ(pair.second, 1.0f);
+    for (int j = 0; j < num_values / 2; ++j) {
+      std::complex<float> cf(output_floats[j * 2], output_floats[j * 2 + 1]);
+      EXPECT_FLOAT_EQ(std::norm(cf), 1.0f);
+    }
+  }
+  CloseGraph();
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensor/bert_preprocessor_calculator.cc

@@ -0,0 +1,251 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/ascii.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/tensor/bert_preprocessor_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/tasks/cc/core/utils.h"
+#include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
+#include "mediapipe/tasks/cc/text/tokenizers/tokenizer.h"
+#include "mediapipe/tasks/cc/text/tokenizers/tokenizer_utils.h"
+#include "mediapipe/tasks/metadata/metadata_schema_generated.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using ::mediapipe::tasks::core::FindTensorIndexByMetadataName;
+using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
+
+constexpr int kNumInputTensorsForBert = 3;
+constexpr int kTokenizerProcessUnitIndex = 0;
+constexpr absl::string_view kInputIdsTensorName = "ids";
+constexpr absl::string_view kInputMasksTensorName = "mask";
+constexpr absl::string_view kSegmentIdsTensorName = "segment_ids";
+constexpr absl::string_view kClassifierToken = "[CLS]";
+constexpr absl::string_view kSeparatorToken = "[SEP]";
+
+// Preprocesses input text into three int32 input tensors for a BERT model using
+// a tokenizer.
+// The associated BERT model is expected to contain input tensors with names:
+//
+// Tensor           | Metadata Name
+// ---------------- | --------------
+// IDs              | "ids"
+// Segment IDs      | "segment_ids"
+// Mask             | "mask"
+//
+// This calculator will return an error if the model does not have three input
+// tensors or if the tensors do not have names corresponding to the above
+// metadata names in some order. Additional details regarding these input
+// tensors are given in the Calculator "Outputs" section below.
+//
+// This calculator is currently configured for the TextClassifier Task but it
+// will eventually be generalized for other Text Tasks.
+// TODO: Handle preprocessing for other Text Tasks too.
+//
+// Inputs:
+//   TEXT - std::string
+//     The input text.
+// Side Inputs:
+//   METADATA_EXTRACTOR - ModelMetadataExtractor
+//     The metadata extractor for the BERT model. Used to determine the order of
+//     the three input Tensors for the BERT model and to extract the metadata to
+//     construct the tokenizer.
+//
+// Outputs:
+//   TENSORS - std::vector<Tensor>
+//     Vector containing the three input Tensors for the BERT model:
+//       (1): the token ids of the tokenized input string. A classifier token
+//            ("[CLS]") will be prepended to the input tokens and a separator
+//            token ("[SEP]") will be appended to the input tokens.
+//       (2): the segment ids, which are all 0 for now but will have different
+//            values to distinguish between different sentences in the input
+//            text for other Text tasks.
+//       (3): the input mask ids, which are 1 at each of the input token indices
+//            and 0 elsewhere.
+//     The Tensors will have size equal to the max sequence length for the BERT
+//     model.
+//
+// Example:
+// node {
+//   calculator: "BertPreprocessorCalculator"
+//   input_stream: "TEXT:text"
+//   input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
+//   output_stream: "TENSORS:tensors"
+//   options {
+//     [mediapipe.BertPreprocessorCalculatorOptions.ext] {
+//       bert_max_seq_len: 128
+//     }
+//   }
+// }
+class BertPreprocessorCalculator : public Node {
+ public:
+  static constexpr Input<std::string> kTextIn{"TEXT"};
+  static constexpr SideInput<ModelMetadataExtractor> kMetadataExtractorSideIn{
+      "METADATA_EXTRACTOR"};
+  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
+
+  MEDIAPIPE_NODE_CONTRACT(kTextIn, kMetadataExtractorSideIn, kTensorsOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  std::unique_ptr<tasks::text::tokenizers::Tokenizer> tokenizer_;
+  // The max sequence length accepted by the BERT model.
+  int bert_max_seq_len_ = 2;
+  // Indices of the three input tensors for the BERT model. They should form the
+  // set {0, 1, 2}.
+  int input_ids_tensor_index_ = 0;
+  int segment_ids_tensor_index_ = 1;
+  int input_masks_tensor_index_ = 2;
+
+  // Applies `tokenizer_` to the `input_text` to generate a vector of tokens.
+  // This util prepends "[CLS]" and appends "[SEP]" to the input tokens and
+  // clips the vector of tokens to have length at most `bert_max_seq_len_`.
+  std::vector<std::string> TokenizeInputText(absl::string_view input_text);
+  // Processes the `input_tokens` to generate the three input tensors for the
+  // BERT model.
+  std::vector<Tensor> GenerateInputTensors(
+      const std::vector<std::string>& input_tokens);
+};
+
+absl::Status BertPreprocessorCalculator::UpdateContract(
+    CalculatorContract* cc) {
+  const auto& options =
+      cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
+  RET_CHECK(options.has_bert_max_seq_len()) << "bert_max_seq_len is required";
+  RET_CHECK_GE(options.bert_max_seq_len(), 2)
+      << "bert_max_seq_len must be at least 2";
+  return absl::OkStatus();
+}
+
+absl::Status BertPreprocessorCalculator::Open(CalculatorContext* cc) {
+  const ModelMetadataExtractor* metadata_extractor =
+      &kMetadataExtractorSideIn(cc).Get();
+  const tflite::ProcessUnit* tokenizer_metadata =
+      metadata_extractor->GetInputProcessUnit(kTokenizerProcessUnitIndex);
+  ASSIGN_OR_RETURN(tokenizer_,
+                   tasks::text::tokenizers::CreateTokenizerFromProcessUnit(
+                       tokenizer_metadata, metadata_extractor));
+
+  auto* input_tensors_metadata = metadata_extractor->GetInputTensorMetadata();
+  input_ids_tensor_index_ = FindTensorIndexByMetadataName(
+      input_tensors_metadata, kInputIdsTensorName);
+  segment_ids_tensor_index_ = FindTensorIndexByMetadataName(
+      input_tensors_metadata, kSegmentIdsTensorName);
+  input_masks_tensor_index_ = FindTensorIndexByMetadataName(
+      input_tensors_metadata, kInputMasksTensorName);
+  absl::flat_hash_set<int> tensor_indices = {input_ids_tensor_index_,
+                                             segment_ids_tensor_index_,
+                                             input_masks_tensor_index_};
+  if (tensor_indices != absl::flat_hash_set<int>({0, 1, 2})) {
+    return absl::InvalidArgumentError(absl::Substitute(
+        "Input tensor indices form the set {$0, $1, $2} rather than {0, 1, 2}",
+        input_ids_tensor_index_, segment_ids_tensor_index_,
+        input_masks_tensor_index_));
+  }
+
+  const auto& options =
+      cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
+  bert_max_seq_len_ = options.bert_max_seq_len();
+  return absl::OkStatus();
+}
+
+absl::Status BertPreprocessorCalculator::Process(CalculatorContext* cc) {
+  kTensorsOut(cc).Send(
+      GenerateInputTensors(TokenizeInputText(kTextIn(cc).Get())));
+  return absl::OkStatus();
+}
+
+std::vector<std::string> BertPreprocessorCalculator::TokenizeInputText(
+    absl::string_view input_text) {
+  std::string processed_input = std::string(input_text);
+  absl::AsciiStrToLower(&processed_input);
+
+  tasks::text::tokenizers::TokenizerResult tokenizer_result =
+      tokenizer_->Tokenize(processed_input);
+
+  // Offset by 2 to account for [CLS] and [SEP]
+  int input_tokens_size =
+      std::min(bert_max_seq_len_,
+               static_cast<int>(tokenizer_result.subwords.size()) + 2);
+  std::vector<std::string> input_tokens;
+  input_tokens.reserve(input_tokens_size);
+  input_tokens.push_back(std::string(kClassifierToken));
+  for (int i = 0; i < input_tokens_size - 2; ++i) {
+    input_tokens.push_back(std::move(tokenizer_result.subwords[i]));
+  }
+  input_tokens.push_back(std::string(kSeparatorToken));
+  return input_tokens;
+}
+
+std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
+    const std::vector<std::string>& input_tokens) {
+  std::vector<int32_t> input_ids(bert_max_seq_len_, 0);
+  std::vector<int32_t> segment_ids(bert_max_seq_len_, 0);
+  std::vector<int32_t> input_masks(bert_max_seq_len_, 0);
+  // Convert tokens back into ids and set mask
+  for (int i = 0; i < input_tokens.size(); ++i) {
+    tokenizer_->LookupId(input_tokens[i], &input_ids[i]);
+    input_masks[i] = 1;
+  }
+  //                           |<--------bert_max_seq_len_--------->|
+  // input_ids                 [CLS] s1  s2...  sn [SEP]  0  0...  0
+  // segment_ids                 0    0   0...  0    0    0  0...  0
+  // input_masks                 1    1   1...  1    1    0  0...  0
+
+  std::vector<Tensor> input_tensors;
+  input_tensors.reserve(kNumInputTensorsForBert);
+  for (int i = 0; i < kNumInputTensorsForBert; ++i) {
+    input_tensors.push_back(
+        {Tensor::ElementType::kInt32, Tensor::Shape({bert_max_seq_len_})});
+  }
+  std::memcpy(input_tensors[input_ids_tensor_index_]
+                  .GetCpuWriteView()
+                  .buffer<int32_t>(),
+              input_ids.data(), input_ids.size() * sizeof(int32_t));
+  std::memcpy(input_tensors[segment_ids_tensor_index_]
+                  .GetCpuWriteView()
+                  .buffer<int32_t>(),
+              segment_ids.data(), segment_ids.size() * sizeof(int32_t));
+  std::memcpy(input_tensors[input_masks_tensor_index_]
+                  .GetCpuWriteView()
+                  .buffer<int32_t>(),
+              input_masks.data(), input_masks.size() * sizeof(int32_t));
+  return input_tensors;
+}
+
+MEDIAPIPE_REGISTER_NODE(BertPreprocessorCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/bert_preprocessor_calculator.proto

@@ -0,0 +1,29 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message BertPreprocessorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional BertPreprocessorCalculatorOptions ext = 462509271;
+  }
+
+  // The maximum input sequence length for the calculator's BERT model.
+  optional int32 bert_max_seq_len = 1;
+}

mediapipe/calculators/tensor/bert_preprocessor_calculator_test.cc

@@ -0,0 +1,154 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/tasks/cc/core/utils.h"
+#include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
+using ::testing::ElementsAreArray;
+
+constexpr int kNumInputTensorsForBert = 3;
+constexpr int kBertMaxSeqLen = 128;
+constexpr absl::string_view kTestModelPath =
+    "mediapipe/tasks/testdata/text/bert_text_classifier.tflite";
+
+absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
+    absl::string_view text, absl::string_view model_path) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
+      absl::Substitute(R"(
+        input_stream: "text"
+        output_stream: "tensors"
+        node {
+          calculator: "BertPreprocessorCalculator"
+          input_stream: "TEXT:text"
+          input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
+          output_stream: "TENSORS:tensors"
+          options {
+            [mediapipe.BertPreprocessorCalculatorOptions.ext] {
+              bert_max_seq_len: $0
+            }
+          }
+        }
+      )",
+                       kBertMaxSeqLen));
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("tensors", &graph_config, &output_packets);
+
+  std::string model_buffer = tasks::core::LoadBinaryContent(model_path.data());
+  ASSIGN_OR_RETURN(std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
+                   ModelMetadataExtractor::CreateFromModelBuffer(
+                       model_buffer.data(), model_buffer.size()));
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_RETURN_IF_ERROR(graph.Initialize(
+      graph_config,
+      {{"metadata_extractor",
+        MakePacket<ModelMetadataExtractor>(std::move(*metadata_extractor))}}));
+  MP_RETURN_IF_ERROR(graph.StartRun({}));
+  MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
+      "text", MakePacket<std::string>(text).At(Timestamp(0))));
+  MP_RETURN_IF_ERROR(graph.WaitUntilIdle());
+
+  if (output_packets.size() != 1) {
+    return absl::InvalidArgumentError(absl::Substitute(
+        "output_packets has size $0, expected 1", output_packets.size()));
+  }
+  const std::vector<Tensor>& tensor_vec =
+      output_packets[0].Get<std::vector<Tensor>>();
+  if (tensor_vec.size() != kNumInputTensorsForBert) {
+    return absl::InvalidArgumentError(
+        absl::Substitute("tensor_vec has size $0, expected $1",
+                         tensor_vec.size(), kNumInputTensorsForBert));
+  }
+
+  std::vector<std::vector<int>> results;
+  for (int i = 0; i < kNumInputTensorsForBert; i++) {
+    const Tensor& tensor = tensor_vec[i];
+    if (tensor.element_type() != Tensor::ElementType::kInt32) {
+      return absl::InvalidArgumentError("Expected tensor element type kInt32");
+    }
+    auto* buffer = tensor.GetCpuReadView().buffer<int>();
+    std::vector<int> buffer_view(buffer, buffer + kBertMaxSeqLen);
+    results.push_back(buffer_view);
+  }
+  MP_RETURN_IF_ERROR(graph.CloseAllPacketSources());
+  MP_RETURN_IF_ERROR(graph.WaitUntilDone());
+  return results;
+}
+
+TEST(BertPreprocessorCalculatorTest, TextClassifierWithBertModel) {
+  std::vector<std::vector<int>> expected_result = {
+      {101, 2009, 1005, 1055, 1037, 11951, 1998, 2411, 12473, 4990, 102}};
+  // segment_ids
+  expected_result.push_back(std::vector(kBertMaxSeqLen, 0));
+  // input_masks
+  expected_result.push_back(std::vector(expected_result[0].size(), 1));
+  expected_result[2].resize(kBertMaxSeqLen);
+  // padding input_ids
+  expected_result[0].resize(kBertMaxSeqLen);
+
+  MP_ASSERT_OK_AND_ASSIGN(
+      std::vector<std::vector<int>> processed_tensor_values,
+      RunBertPreprocessorCalculator(
+          "it's a charming and often affecting journey", kTestModelPath));
+  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
+}
+
+TEST(BertPreprocessorCalculatorTest, LongInput) {
+  std::stringstream long_input;
+  long_input
+      << "it's a charming and often affecting journey and this is a long";
+  for (int i = 0; i < kBertMaxSeqLen; ++i) {
+    long_input << " long";
+  }
+  long_input << " movie review";
+  std::vector<std::vector<int>> expected_result = {
+      {101, 2009, 1005, 1055, 1037, 11951, 1998, 2411, 12473, 4990, 1998, 2023,
+       2003, 1037}};
+  // "long" id
+  expected_result[0].resize(kBertMaxSeqLen - 1, 2146);
+  // "[SEP]" id
+  expected_result[0].push_back(102);
+  // segment_ids
+  expected_result.push_back(std::vector(kBertMaxSeqLen, 0));
+  // input_masks
+  expected_result.push_back(std::vector(kBertMaxSeqLen, 1));
+
+  MP_ASSERT_OK_AND_ASSIGN(
+      std::vector<std::vector<int>> processed_tensor_values,
+      RunBertPreprocessorCalculator(long_input.str(), kTestModelPath));
+  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/feedback_tensors_calculator.cc

@@ -0,0 +1,165 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+
+#include "absl/status/status.h"
+#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+constexpr char kInputTensorsTag[] = "INPUT_TENSORS";
+constexpr char kFeedbackTensorsTag[] = "FEEDBACK_TENSORS";
+constexpr char kOutputTensorsTag[] = "TENSORS";
+
+using Tensors = std::vector<Tensor>;
+}  // namespace
+
+// FeedbackTensorsCalculator groups the input and the feedback (typically
+// recurrent neural network cell state output tensors from the previous run)
+// tensor vectors as the input tensor vector for the next recurrent model cell
+// inference. On the first step, the feedback tensor is filled with zeros to
+// jumpstart the loop.
+class FeedbackTensorsCalculator : public Node {
+ public:
+  static constexpr Input<Tensors> kFeedbackTensorsIn{kFeedbackTensorsTag};
+  static constexpr Input<Tensors> kInputTensorsIn{kInputTensorsTag};
+  static constexpr Output<Tensors> kTensorsOut{kOutputTensorsTag};
+
+  MEDIAPIPE_NODE_CONTRACT(kFeedbackTensorsIn, kInputTensorsIn, kTensorsOut);
+
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->SetProcessTimestampBounds(true);
+    return absl::OkStatus();
+  }
+
+  absl::Status Open(CalculatorContext* cc) override {
+    const auto& options =
+        cc->Options<mediapipe::FeedbackTensorsCalculatorOptions>();
+
+    const auto& shape_dims = options.feedback_tensor_shape().dims();
+    feedback_tensor_shape_.dims.assign(shape_dims.begin(), shape_dims.end());
+    feedback_tensor_size_ = feedback_tensor_shape_.num_elements();
+
+    num_feedback_tensors_ = options.num_feedback_tensors();
+
+    feedback_tensors_location_ = options.location();
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    if (feedback_tensors_location_ ==
+        mediapipe::FeedbackTensorsCalculatorOptions::NONE) {
+      kTensorsOut(cc).Send(kInputTensorsIn(cc).packet().As<Tensors>());
+      return absl::OkStatus();
+    }
+
+    std::vector<Tensor> outputs;
+    switch (feedback_tensors_location_) {
+      case mediapipe::FeedbackTensorsCalculatorOptions::PREPENDED:
+        MP_RETURN_IF_ERROR(AddFeedbackTensors(cc, outputs));
+        MP_RETURN_IF_ERROR(AddInputTensors(cc, outputs));
+        break;
+      case mediapipe::FeedbackTensorsCalculatorOptions::APPENDED:
+        MP_RETURN_IF_ERROR(AddInputTensors(cc, outputs));
+        MP_RETURN_IF_ERROR(AddFeedbackTensors(cc, outputs));
+        break;
+      default:
+        return absl::InvalidArgumentError(
+            "Unsupported feedback tensors location");
+    }
+    kTensorsOut(cc).Send(std::move(outputs));
+    return absl::OkStatus();
+  }
+
+ private:
+  absl::Status AddInputTensors(CalculatorContext* cc,
+                               std::vector<Tensor>& outputs) {
+    absl::StatusOr<std::unique_ptr<std::vector<Tensor>>> input_tensors =
+        cc->Inputs()
+            .Tag(kInputTensorsTag)
+            .Value()
+            .Consume<std::vector<Tensor>>();
+    if (!input_tensors.ok()) {
+      return absl::InternalError("The input tensors packet is not consumable");
+    }
+    RET_CHECK(*input_tensors);
+    std::vector<Tensor>& inputs = **input_tensors;
+    outputs.insert(outputs.end(), std::make_move_iterator(inputs.begin()),
+                   std::make_move_iterator(inputs.end()));
+    return absl::OkStatus();
+  }
+
+  absl::Status AddFeedbackTensors(CalculatorContext* cc,
+                                  std::vector<Tensor>& outputs) {
+    if (first_run_) {
+      for (int index = 0; index < num_feedback_tensors_; ++index) {
+        Tensor initial_feedback_tensor(Tensor::ElementType::kFloat32,
+                                       feedback_tensor_shape_);
+        float* data = initial_feedback_tensor.GetCpuWriteView().buffer<float>();
+        std::fill_n(data, feedback_tensor_size_, 0.0f);
+        outputs.push_back(std::move(initial_feedback_tensor));
+      }
+      first_run_ = false;
+      return absl::OkStatus();
+    }
+
+    if (num_feedback_tensors_ != kFeedbackTensorsIn(cc)->size()) {
+      return absl::InvalidArgumentError(
+          "The number of tensors fed back differs from the configuration");
+    }
+    absl::StatusOr<std::unique_ptr<std::vector<Tensor>>> feedback_tensors =
+        cc->Inputs()
+            .Tag(kFeedbackTensorsTag)
+            .Value()
+            .Consume<std::vector<Tensor>>();
+    if (!feedback_tensors.ok()) {
+      return absl::InternalError(
+          "The feedback tensors packet is not consumable");
+    }
+    RET_CHECK(*feedback_tensors);
+    std::vector<Tensor>& feedbacks = **feedback_tensors;
+    for (const auto& feedback : feedbacks) {
+      if (feedback.shape().dims != feedback_tensor_shape_.dims) {
+        return absl::InvalidArgumentError(
+            "The shape of a tensor fed back differs from the configuration");
+      }
+    }
+    outputs.insert(outputs.end(), std::make_move_iterator(feedbacks.begin()),
+                   std::make_move_iterator(feedbacks.end()));
+
+    return absl::OkStatus();
+  }
+
+  Tensor::Shape feedback_tensor_shape_;
+  int num_feedback_tensors_ = 0;
+  mediapipe::FeedbackTensorsCalculatorOptions::FeedbackTensorsLocation
+      feedback_tensors_location_;
+
+  int feedback_tensor_size_ = 0;
+  bool first_run_ = true;
+};
+
+MEDIAPIPE_REGISTER_NODE(FeedbackTensorsCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/feedback_tensors_calculator.proto

@@ -0,0 +1,47 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FeedbackTensorsCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional FeedbackTensorsCalculatorOptions ext = 474496252;
+  }
+
+  // Represents the dimensions of a tensor starting from the outermost size.
+  message TensorShape {
+    repeated int32 dims = 1 [packed = true];
+  }
+
+  // The shape of the feedback tensors to add.
+  optional TensorShape feedback_tensor_shape = 1;
+  // The number of the feedback tensors to add.
+  optional int32 num_feedback_tensors = 2 [default = 1];
+
+  enum FeedbackTensorsLocation {
+    // The feedback tensors will not be added.
+    NONE = 0;
+    // The feedback tensors will be added before the input tensors.
+    PREPENDED = 1;
+    // The feedback tensors will be added after the input tensors.
+    APPENDED = 2;
+  }
+
+  // Determines the location of the feedback tensor(s) in the output vector.
+  optional FeedbackTensorsLocation location = 3 [default = APPENDED];
+}

mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc

@@ -0,0 +1,389 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <functional>
+#include <initializer_list>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::CalculatorGraphConfig;
+using ::testing::ElementsAreArray;
+using ::testing::Not;
+using Tensors = std::vector<Tensor>;
+
+template <typename T>
+struct TensorElementType {
+  static constexpr Tensor::ElementType value = Tensor::ElementType::kNone;
+};
+
+template <>
+struct TensorElementType<float> {
+  static constexpr Tensor::ElementType value = Tensor::ElementType::kFloat32;
+};
+
+template <>
+struct TensorElementType<std::int8_t> {
+  static constexpr Tensor::ElementType value = Tensor::ElementType::kInt8;
+};
+
+template <>
+struct TensorElementType<std::uint8_t> {
+  static constexpr Tensor::ElementType value = Tensor::ElementType::kUInt8;
+};
+
+template <>
+struct TensorElementType<std::int32_t> {
+  static constexpr Tensor::ElementType value = Tensor::ElementType::kInt32;
+};
+
+template <typename T>
+Tensor MakeTensor(std::initializer_list<int> shape,
+                  std::initializer_list<T> values) {
+  Tensor tensor(TensorElementType<T>::value, shape);
+  CHECK_EQ(values.size(), tensor.shape().num_elements())
+      << "The size of `values` is incompatible with `shape`";
+  absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
+  return tensor;
+}
+
+template <typename T>
+void ValidateTensor(const Tensor& tensor,
+                    const std::vector<int>& expected_shape,
+                    const std::vector<T>& expected_values) {
+  ASSERT_EQ(tensor.element_type(), TensorElementType<T>::value);
+  EXPECT_EQ(tensor.shape().dims, expected_shape);
+  EXPECT_EQ(tensor.shape().num_elements(), expected_values.size());
+
+  auto* tensor_buffer = tensor.GetCpuReadView().buffer<T>();
+  const std::vector<T> tensor_values(
+      tensor_buffer, tensor_buffer + tensor.shape().num_elements());
+  EXPECT_THAT(tensor_values, ElementsAreArray(expected_values));
+}
+
+TEST(FeedbackTensorsCalculatorTest, AppendsFeedback) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    input_stream: "feedback"
+    node {
+      calculator: "FeedbackTensorsCalculator"
+      input_stream: "INPUT_TENSORS:input"
+      input_stream: "FEEDBACK_TENSORS:feedback"
+      output_stream: "TENSORS:output"
+      options: {
+        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
+          feedback_tensor_shape: { dims: 2 dims: 3 }
+          location: APPENDED
+        }
+      }
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto initial_input_tensors = std::make_unique<Tensors>();
+  initial_input_tensors->push_back(
+      MakeTensor<std::int32_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
+  // At the beginning, the loopback packet with the model feedback is missing.
+  // The calculator has to assume it's all-zero with the shape from the options.
+
+  auto later_input_tensors = std::make_unique<Tensors>();
+  later_input_tensors->push_back(
+      MakeTensor<std::int32_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
+  auto later_feedback_tensors = std::make_unique<Tensors>();
+  later_feedback_tensors->push_back(
+      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams())
+      << "Couldn't close the graph inputs";
+  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
+
+  ASSERT_EQ(output_packets.size(), 2);
+
+  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
+  ASSERT_EQ(initial_combined_tensors.size(), 2);
+  ValidateTensor<std::int32_t>(initial_combined_tensors[0],
+                               /*expected_shape=*/{2, 4},
+                               /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
+  // The initial feedback is zero.
+  ValidateTensor<float>(initial_combined_tensors[1], /*expected_shape=*/{2, 3},
+                        /*expected_values=*/{0.f, 0.f, 0.f, 0.f, 0.f, 0.f});
+
+  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
+  ASSERT_EQ(later_combined_tensors.size(), 2);
+  ValidateTensor<std::int32_t>(later_combined_tensors[0],
+                               /*expected_shape=*/{2, 4},
+                               /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
+  // Afterwards, the provided feedback is passed through.
+  ValidateTensor<float>(
+      later_combined_tensors[1], /*expected_shape=*/{2, 3},
+      /*expected_values=*/{-1.f, -2.f, -3.f, -4.f, -5.f, -6.f});
+}
+
+TEST(FeedbackTensorsCalculatorTest, PrependsFeedback) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    input_stream: "feedback"
+    node {
+      calculator: "FeedbackTensorsCalculator"
+      input_stream: "INPUT_TENSORS:input"
+      input_stream: "FEEDBACK_TENSORS:feedback"
+      output_stream: "TENSORS:output"
+      options: {
+        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
+          feedback_tensor_shape: { dims: 3 dims: 2 }
+          location: PREPENDED
+        }
+      }
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto initial_input_tensors = std::make_unique<Tensors>();
+  initial_input_tensors->push_back(
+      MakeTensor<std::int8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
+  // At the beginning, the loopback packet with the model feedback is missing.
+  // The calculator has to assume it's all-zero with the shape from the options.
+
+  auto later_input_tensors = std::make_unique<Tensors>();
+  later_input_tensors->push_back(
+      MakeTensor<std::int8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
+  auto later_feedback_tensors = std::make_unique<Tensors>();
+  later_feedback_tensors->push_back(
+      MakeTensor({3, 2}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams())
+      << "Couldn't close the graph inputs";
+  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
+
+  ASSERT_EQ(output_packets.size(), 2);
+
+  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
+  ASSERT_EQ(initial_combined_tensors.size(), 2);
+  // The initial feedback is zero.
+  ValidateTensor<float>(initial_combined_tensors[0], /*expected_shape=*/{3, 2},
+                        /*expected_values=*/{0.f, 0.f, 0.f, 0.f, 0.f, 0.f});
+  ValidateTensor<std::int8_t>(initial_combined_tensors[1],
+                              /*expected_shape=*/{2, 4},
+                              /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
+
+  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
+  ASSERT_EQ(later_combined_tensors.size(), 2);
+  // Afterwards, the provided feedback is passed through.
+  ValidateTensor<float>(
+      later_combined_tensors[0], /*expected_shape=*/{3, 2},
+      /*expected_values=*/{-1.f, -2.f, -3.f, -4.f, -5.f, -6.f});
+  ValidateTensor<std::int8_t>(later_combined_tensors[1],
+                              /*expected_shape=*/{2, 4},
+                              /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
+}
+
+TEST(FeedbackTensorsCalculatorTest, NoFeedback) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    input_stream: "feedback"
+    node {
+      calculator: "FeedbackTensorsCalculator"
+      input_stream: "INPUT_TENSORS:input"
+      input_stream: "FEEDBACK_TENSORS:feedback"
+      output_stream: "TENSORS:output"
+      options: {
+        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
+          feedback_tensor_shape: { dims: 3 dims: 4 }
+          location: NONE
+        }
+      }
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto initial_input_tensors = std::make_unique<Tensors>();
+  initial_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
+  // At the beginning, the loopback packet with the model feedback is missing.
+
+  auto later_input_tensors = std::make_unique<Tensors>();
+  later_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
+  // This feedback should be ignored due to `location: NONE`.
+  auto later_feedback_tensors = std::make_unique<Tensors>();
+  later_feedback_tensors->push_back(
+      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams())
+      << "Couldn't close the graph inputs";
+  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
+
+  ASSERT_EQ(output_packets.size(), 2);
+
+  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
+  ASSERT_EQ(initial_combined_tensors.size(), 1);
+  ValidateTensor<std::uint8_t>(initial_combined_tensors[0],
+                               /*expected_shape=*/{2, 4},
+                               /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
+  // No feedback due to `location: NONE`.
+
+  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
+  ASSERT_EQ(later_combined_tensors.size(), 1);
+  ValidateTensor<std::uint8_t>(later_combined_tensors[0],
+                               /*expected_shape=*/{2, 4},
+                               /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
+}
+
+TEST(FeedbackTensorsCalculatorTest, ChecksTensorNumber) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    input_stream: "feedback"
+    node {
+      calculator: "FeedbackTensorsCalculator"
+      input_stream: "INPUT_TENSORS:input"
+      input_stream: "FEEDBACK_TENSORS:feedback"
+      output_stream: "TENSORS:output"
+      options: {
+        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
+          num_feedback_tensors: 2
+          feedback_tensor_shape: { dims: 2 dims: 3 }
+          location: PREPENDED
+        }
+      }
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto initial_input_tensors = std::make_unique<Tensors>();
+  initial_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
+  // At the beginning, the loopback packet with the model feedback is missing.
+
+  auto later_input_tensors = std::make_unique<Tensors>();
+  later_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
+  // This feedback should be ignored due to `location: NONE`.
+  auto later_feedback_tensors = std::make_unique<Tensors>();
+  later_feedback_tensors->push_back(
+      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams())
+      << "Couldn't close the graph inputs";
+  EXPECT_THAT(graph.WaitUntilDone(), Not(IsOk()))
+      << "Tensor number mismatch missed";
+}
+
+TEST(FeedbackTensorsCalculatorTest, ChecksShape) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    input_stream: "feedback"
+    node {
+      calculator: "FeedbackTensorsCalculator"
+      input_stream: "INPUT_TENSORS:input"
+      input_stream: "FEEDBACK_TENSORS:feedback"
+      output_stream: "TENSORS:output"
+      options: {
+        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
+          feedback_tensor_shape: { dims: 3 dims: 4 }
+          location: APPENDED
+        }
+      }
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto initial_input_tensors = std::make_unique<Tensors>();
+  initial_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
+  // At the beginning, the loopback packet with the model feedback is missing.
+
+  auto later_input_tensors = std::make_unique<Tensors>();
+  later_input_tensors->push_back(
+      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
+  // This feedback should be ignored due to `location: NONE`.
+  auto later_feedback_tensors = std::make_unique<Tensors>();
+  later_feedback_tensors->push_back(
+      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams())
+      << "Couldn't close the graph inputs";
+  EXPECT_THAT(graph.WaitUntilDone(), Not(IsOk()))
+      << "Tensor shape mismatch missed";
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -54,13 +54,6 @@
 namespace mediapipe {
 namespace api2 {
 
-#if MEDIAPIPE_DISABLE_GPU
-// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
-using GpuBuffer = AnyType;
-#else
-using GpuBuffer = mediapipe::GpuBuffer;
-#endif  // MEDIAPIPE_DISABLE_GPU
-
 // Converts image into Tensor, possibly with cropping, resizing and
 // normalization, according to specified inputs and options.
 //
@@ -141,42 +134,7 @@ class ImageToTensorCalculator : public Node {
     const auto& options =
         cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
 
-    RET_CHECK(options.has_output_tensor_float_range() ||
-              options.has_output_tensor_int_range() ||
-              options.has_output_tensor_uint_range())
-        << "Output tensor range is required.";
-    if (options.has_output_tensor_float_range()) {
-      RET_CHECK_LT(options.output_tensor_float_range().min(),
-                   options.output_tensor_float_range().max())
-          << "Valid output float tensor range is required.";
-    }
-    if (options.has_output_tensor_uint_range()) {
-      RET_CHECK_LT(options.output_tensor_uint_range().min(),
-                   options.output_tensor_uint_range().max())
-          << "Valid output uint tensor range is required.";
-      RET_CHECK_GE(options.output_tensor_uint_range().min(), 0)
-          << "The minimum of the output uint tensor range must be "
-             "non-negative.";
-      RET_CHECK_LE(options.output_tensor_uint_range().max(), 255)
-          << "The maximum of the output uint tensor range must be less than or "
-             "equal to 255.";
-    }
-    if (options.has_output_tensor_int_range()) {
-      RET_CHECK_LT(options.output_tensor_int_range().min(),
-                   options.output_tensor_int_range().max())
-          << "Valid output int tensor range is required.";
-      RET_CHECK_GE(options.output_tensor_int_range().min(), -128)
-          << "The minimum of the output int tensor range must be greater than "
-             "or equal to -128.";
-      RET_CHECK_LE(options.output_tensor_int_range().max(), 127)
-          << "The maximum of the output int tensor range must be less than or "
-             "equal to 127.";
-    }
-    RET_CHECK_GT(options.output_tensor_width(), 0)
-        << "Valid output tensor width is required.";
-    RET_CHECK_GT(options.output_tensor_height(), 0)
-        << "Valid output tensor height is required.";
-
+    RET_CHECK_OK(ValidateOptionOutputDims(options));
     RET_CHECK(kIn(cc).IsConnected() ^ kInGpu(cc).IsConnected())
         << "One and only one of IMAGE and IMAGE_GPU input is expected.";
 
@@ -198,21 +156,7 @@ class ImageToTensorCalculator : public Node {
 
   absl::Status Open(CalculatorContext* cc) {
     options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
-    output_width_ = options_.output_tensor_width();
-    output_height_ = options_.output_tensor_height();
-    is_float_output_ = options_.has_output_tensor_float_range();
-    if (options_.has_output_tensor_uint_range()) {
-      range_min_ =
-          static_cast<float>(options_.output_tensor_uint_range().min());
-      range_max_ =
-          static_cast<float>(options_.output_tensor_uint_range().max());
-    } else if (options_.has_output_tensor_int_range()) {
-      range_min_ = static_cast<float>(options_.output_tensor_int_range().min());
-      range_max_ = static_cast<float>(options_.output_tensor_int_range().max());
-    } else {
-      range_min_ = options_.output_tensor_float_range().min();
-      range_max_ = options_.output_tensor_float_range().max();
-    }
+    params_ = GetOutputTensorParams(options_);
     return absl::OkStatus();
   }
 
@@ -242,9 +186,15 @@ class ImageToTensorCalculator : public Node {
       }
     }
 
-    ASSIGN_OR_RETURN(auto image, GetInputImage(cc));
-    const Size size{image->width(), image->height()};
-    RotatedRect roi = GetRoi(size.width, size.height, norm_rect);
+#if MEDIAPIPE_DISABLE_GPU
+    ASSIGN_OR_RETURN(auto image, GetInputImage(kIn(cc)));
+#else
+    const bool is_input_gpu = kInGpu(cc).IsConnected();
+    ASSIGN_OR_RETURN(auto image, is_input_gpu ? GetInputImage(kInGpu(cc))
+                                              : GetInputImage(kIn(cc)));
+#endif  // MEDIAPIPE_DISABLE_GPU
+
+    RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect);
     ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(),
                                           options_.output_tensor_height(),
                                           options_.keep_aspect_ratio(), &roi));
@@ -253,19 +203,24 @@ class ImageToTensorCalculator : public Node {
     }
     if (kOutMatrix(cc).IsConnected()) {
       std::array<float, 16> matrix;
-      GetRotatedSubRectToRectTransformMatrix(roi, size.width, size.height,
-                                             /*flip_horizontaly=*/false,
-                                             &matrix);
+      GetRotatedSubRectToRectTransformMatrix(
+          roi, image->width(), image->height(),
+          /*flip_horizontaly=*/false, &matrix);
       kOutMatrix(cc).Send(std::move(matrix));
     }
 
     // Lazy initialization of the GPU or CPU converter.
     MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, *image.get()));
 
-    ASSIGN_OR_RETURN(Tensor tensor,
-                     (image->UsesGpu() ? gpu_converter_ : cpu_converter_)
-                         ->Convert(*image, roi, {output_width_, output_height_},
-                                   range_min_, range_max_));
+    Tensor::ElementType output_tensor_type =
+        GetOutputTensorType(image->UsesGpu(), params_);
+    Tensor tensor(output_tensor_type,
+                  {1, params_.output_height, params_.output_width,
+                   GetNumOutputChannels(*image)});
+    MP_RETURN_IF_ERROR((image->UsesGpu() ? gpu_converter_ : cpu_converter_)
+                           ->Convert(*image, roi, params_.range_min,
+                                     params_.range_max,
+                                     /*tensor_buffer_offset=*/0, tensor));
 
     auto result = std::make_unique<std::vector<Tensor>>();
     result->push_back(std::move(tensor));
@@ -275,65 +230,11 @@ class ImageToTensorCalculator : public Node {
   }
 
  private:
-  bool DoesGpuInputStartAtBottom() {
-    return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
-  }
-
-  BorderMode GetBorderMode() {
-    switch (options_.border_mode()) {
-      case mediapipe::
-          ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED:
-        return BorderMode::kReplicate;
-      case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO:
-        return BorderMode::kZero;
-      case mediapipe::
-          ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE:
-        return BorderMode::kReplicate;
-    }
-  }
-
-  Tensor::ElementType GetOutputTensorType() {
-    if (is_float_output_) {
-      return Tensor::ElementType::kFloat32;
-    }
-    if (range_min_ < 0) {
-      return Tensor::ElementType::kInt8;
-    } else {
-      return Tensor::ElementType::kUInt8;
-    }
-  }
-
-  absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
-      CalculatorContext* cc) {
-    if (kIn(cc).IsConnected()) {
-      const auto& packet = kIn(cc).packet();
-      return kIn(cc).Visit(
-          [&packet](const mediapipe::Image&) {
-            return SharedPtrWithPacket<mediapipe::Image>(packet);
-          },
-          [&packet](const mediapipe::ImageFrame&) {
-            return std::make_shared<const mediapipe::Image>(
-                std::const_pointer_cast<mediapipe::ImageFrame>(
-                    SharedPtrWithPacket<mediapipe::ImageFrame>(packet)));
-          });
-    } else {  // if (kInGpu(cc).IsConnected())
-#if !MEDIAPIPE_DISABLE_GPU
-      const GpuBuffer& input = *kInGpu(cc);
-      // A shallow copy is okay since the resulting 'image' object is local in
-      // Process(), and thus never outlives 'input'.
-      return std::make_shared<const mediapipe::Image>(input);
-#else
-      return absl::UnimplementedError(
-          "GPU processing is disabled in build flags");
-#endif  // !MEDIAPIPE_DISABLE_GPU
-    }
-  }
-
   absl::Status InitConverterIfNecessary(CalculatorContext* cc,
                                         const Image& image) {
     // Lazy initialization of the GPU or CPU converter.
     if (image.UsesGpu()) {
-      if (!is_float_output_) {
+      if (!params_.is_float_output) {
         return absl::UnimplementedError(
             "ImageToTensorConverter for the input GPU image currently doesn't "
             "support quantization.");
@@ -341,18 +242,20 @@ class ImageToTensorCalculator : public Node {
       if (!gpu_converter_) {
 #if !MEDIAPIPE_DISABLE_GPU
 #if MEDIAPIPE_METAL_ENABLED
-        ASSIGN_OR_RETURN(gpu_converter_,
-                         CreateMetalConverter(cc, GetBorderMode()));
+        ASSIGN_OR_RETURN(
+            gpu_converter_,
+            CreateMetalConverter(cc, GetBorderMode(options_.border_mode())));
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
         ASSIGN_OR_RETURN(gpu_converter_,
                          CreateImageToGlBufferTensorConverter(
-                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(options_),
+                             GetBorderMode(options_.border_mode())));
 #else
         if (!gpu_converter_) {
-          ASSIGN_OR_RETURN(
-              gpu_converter_,
-              CreateImageToGlTextureTensorConverter(
-                  cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+          ASSIGN_OR_RETURN(gpu_converter_,
+                           CreateImageToGlTextureTensorConverter(
+                               cc, DoesGpuInputStartAtBottom(options_),
+                               GetBorderMode(options_.border_mode())));
         }
         if (!gpu_converter_) {
           return absl::UnimplementedError(
@@ -364,9 +267,10 @@ class ImageToTensorCalculator : public Node {
     } else {
       if (!cpu_converter_) {
 #if !MEDIAPIPE_DISABLE_OPENCV
-        ASSIGN_OR_RETURN(
-            cpu_converter_,
-            CreateOpenCvConverter(cc, GetBorderMode(), GetOutputTensorType()));
+        ASSIGN_OR_RETURN(cpu_converter_,
+                         CreateOpenCvConverter(
+                             cc, GetBorderMode(options_.border_mode()),
+                             GetOutputTensorType(/*uses_gpu=*/false, params_)));
 #else
         LOG(FATAL) << "Cannot create image to tensor opencv converter since "
                       "MEDIAPIPE_DISABLE_OPENCV is defined.";
@@ -379,11 +283,7 @@ class ImageToTensorCalculator : public Node {
   std::unique_ptr<ImageToTensorConverter> gpu_converter_;
   std::unique_ptr<ImageToTensorConverter> cpu_converter_;
   mediapipe::ImageToTensorCalculatorOptions options_;
-  int output_width_ = 0;
-  int output_height_ = 0;
-  bool is_float_output_ = false;
-  float range_min_ = 0.0f;
-  float range_max_ = 1.0f;
+  OutputTensorParams params_;
 };
 
 MEDIAPIPE_REGISTER_NODE(ImageToTensorCalculator);

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -36,22 +36,17 @@
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/image_test_utils.h"
 
 namespace mediapipe {
 namespace {
 
-cv::Mat GetRgb(absl::string_view path) {
-  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
-  cv::Mat rgb;
-  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGB);
-  return rgb;
-}
+constexpr char kTestDataDir[] =
+    "/mediapipe/calculators/tensor/testdata/"
+    "image_to_tensor/";
 
-cv::Mat GetRgba(absl::string_view path) {
-  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
-  cv::Mat rgb;
-  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA);
-  return rgb;
+std::string GetFilePath(absl::string_view filename) {
+  return file::JoinPath("./", kTestDataDir, filename);
 }
 
 // Image to tensor test template.
@@ -147,29 +142,34 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
   ASSERT_THAT(tensor_vec, testing::SizeIs(1));
 
   const Tensor& tensor = tensor_vec[0];
+  const int channels = tensor.shape().dims[3];
+  ASSERT_TRUE(channels == 1 || channels == 3);
   auto view = tensor.GetCpuReadView();
   cv::Mat tensor_mat;
   if (output_int_tensor) {
     if (range_min < 0) {
       EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kInt8);
-      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8SC3,
+      tensor_mat = cv::Mat(tensor_height, tensor_width,
+                           channels == 1 ? CV_8SC1 : CV_8SC3,
                            const_cast<int8*>(view.buffer<int8>()));
     } else {
       EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kUInt8);
-      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8UC3,
+      tensor_mat = cv::Mat(tensor_height, tensor_width,
+                           channels == 1 ? CV_8UC1 : CV_8UC3,
                            const_cast<uint8*>(view.buffer<uint8>()));
     }
   } else {
     EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
-    tensor_mat = cv::Mat(tensor_height, tensor_width, CV_32FC3,
+    tensor_mat = cv::Mat(tensor_height, tensor_width,
+                         channels == 1 ? CV_32FC1 : CV_32FC3,
                          const_cast<float*>(view.buffer<float>()));
   }
 
   cv::Mat result_rgb;
   auto transformation =
       GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f).value();
-  tensor_mat.convertTo(result_rgb, CV_8UC3, transformation.scale,
-                       transformation.offset);
+  tensor_mat.convertTo(result_rgb, channels == 1 ? CV_8UC1 : CV_8UC3,
+                       transformation.scale, transformation.offset);
 
   cv::Mat diff;
   cv::absdiff(result_rgb, expected_result, diff);
@@ -185,17 +185,27 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
   MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
+mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
+  if (image_channels == 4) {
+    return ImageFormat::SRGBA;
+  } else if (image_channels == 3) {
+    return ImageFormat::SRGB;
+  } else if (image_channels == 1) {
+    return ImageFormat::GRAY8;
+  }
+  CHECK(false) << "Unsupported input image channles: " << image_channels;
+}
+
 Packet MakeImageFramePacket(cv::Mat input) {
-  ImageFrame input_image(
-      input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
-      input.cols, input.rows, input.step, input.data, [](uint8*) {});
+  ImageFrame input_image(GetImageFormat(input.channels()), input.cols,
+                         input.rows, input.step, input.data, [](uint8*) {});
   return MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0));
 }
 
 Packet MakeImagePacket(cv::Mat input) {
   mediapipe::Image input_image(std::make_shared<mediapipe::ImageFrame>(
-      input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
-      input.cols, input.rows, input.step, input.data, [](uint8*) {}));
+      GetImageFormat(input.channels()), input.cols, input.rows, input.step,
+      input.data, [](uint8*) {}));
   return MakePacket<mediapipe::Image>(std::move(input_image)).At(Timestamp(0));
 }
 
@@ -237,15 +247,12 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(0);
-  RunTest(
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/input.jpg"),
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
-      /*float_ranges=*/{{0.0f, 1.0f}},
-      /*int_ranges=*/{{0, 255}, {-128, 127}},
-      /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
-      /*border mode*/ {}, roi);
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect.png")),
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
+          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
+          /*border mode*/ {}, roi);
 }
 
 TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
@@ -255,11 +262,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "medium_sub_rect_keep_aspect_border_zero.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect_border_zero.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -273,11 +277,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(M_PI * 90.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "medium_sub_rect_keep_aspect_with_rotation.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect_with_rotation.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -292,11 +293,9 @@ TEST(ImageToTensorCalculatorTest,
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(M_PI * 90.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "medium_sub_rect_keep_aspect_with_rotation_border_zero.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath(
+              "medium_sub_rect_keep_aspect_with_rotation_border_zero.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -310,16 +309,12 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(M_PI * -45.0f / 180.0f);
-  RunTest(
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/input.jpg"),
-      GetRgb(
-          "/mediapipe/calculators/"
-          "tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
-      /*float_ranges=*/{{-1.0f, 1.0f}},
-      /*int_ranges=*/{{0, 255}, {-128, 127}},
-      /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
-      BorderMode::kReplicate, roi);
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("medium_sub_rect_with_rotation.png")),
+          /*float_ranges=*/{{-1.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
+          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
+          BorderMode::kReplicate, roi);
 }
 
 TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
@@ -329,11 +324,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
   roi.set_width(0.5f);
   roi.set_height(0.5f);
   roi.set_rotation(M_PI * -45.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "medium_sub_rect_with_rotation_border_zero.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("medium_sub_rect_with_rotation_border_zero.png")),
           /*float_ranges=*/{{-1.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
@@ -347,10 +339,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRect) {
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/large_sub_rect.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("large_sub_rect.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
@@ -364,15 +354,12 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) {
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(0);
-  RunTest(
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/input.jpg"),
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"),
-      /*float_ranges=*/{{0.0f, 1.0f}},
-      /*int_ranges=*/{{0, 255}, {-128, 127}},
-      /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
-      BorderMode::kZero, roi);
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("large_sub_rect_border_zero.png")),
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
+          BorderMode::kZero, roi);
 }
 
 TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
@@ -382,15 +369,12 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(0);
-  RunTest(
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/input.jpg"),
-      GetRgb("/mediapipe/calculators/"
-             "tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
-      /*float_ranges=*/{{0.0f, 1.0f}},
-      /*int_ranges=*/{{0, 255}, {-128, 127}},
-      /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
-      BorderMode::kReplicate, roi);
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect.png")),
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
+          BorderMode::kReplicate, roi);
 }
 
 TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
@@ -400,11 +384,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "large_sub_rect_keep_aspect_border_zero.png"),
+  RunTest(GetRgb(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect_border_zero.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -418,11 +399,23 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(M_PI * -15.0f / 180.0f);
-  RunTest(GetRgba("/mediapipe/calculators/"
-                  "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "large_sub_rect_keep_aspect_with_rotation.png"),
+  RunTest(GetRgba(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect_with_rotation.png")),
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
+          /*border_mode=*/{}, roi);
+}
+
+TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotationGray) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(M_PI * -15.0f / 180.0f);
+  RunTest(GetGray(GetFilePath("input.jpg")),
+          GetGray(GetFilePath("large_sub_rect_keep_aspect_with_rotation.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -437,11 +430,26 @@ TEST(ImageToTensorCalculatorTest,
   roi.set_width(1.5f);
   roi.set_height(1.1f);
   roi.set_rotation(M_PI * -15.0f / 180.0f);
-  RunTest(GetRgba("/mediapipe/calculators/"
-                  "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/"
-                 "large_sub_rect_keep_aspect_with_rotation_border_zero.png"),
+  RunTest(GetRgba(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath(
+              "large_sub_rect_keep_aspect_with_rotation_border_zero.png")),
+          /*float_ranges=*/{{0.0f, 1.0f}},
+          /*int_ranges=*/{{0, 255}},
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
+          /*border_mode=*/BorderMode::kZero, roi);
+}
+
+TEST(ImageToTensorCalculatorTest,
+     LargeSubRectKeepAspectWithRotationBorderZeroGray) {
+  mediapipe::NormalizedRect roi;
+  roi.set_x_center(0.5f);
+  roi.set_y_center(0.5f);
+  roi.set_width(1.5f);
+  roi.set_height(1.1f);
+  roi.set_rotation(M_PI * -15.0f / 180.0f);
+  RunTest(GetGray(GetFilePath("input.jpg")),
+          GetGray(GetFilePath(
+              "large_sub_rect_keep_aspect_with_rotation_border_zero.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}},
           /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -455,10 +463,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
   roi.set_width(1.0f);
   roi.set_height(1.0f);
   roi.set_rotation(0);
-  RunTest(GetRgba("/mediapipe/calculators/"
-                  "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/noop_except_range.png"),
+  RunTest(GetRgba(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("noop_except_range.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -472,10 +478,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) {
   roi.set_width(1.0f);
   roi.set_height(1.0f);
   roi.set_rotation(0);
-  RunTest(GetRgba("/mediapipe/calculators/"
-                  "tensor/testdata/image_to_tensor/input.jpg"),
-          GetRgb("/mediapipe/calculators/"
-                 "tensor/testdata/image_to_tensor/noop_except_range.png"),
+  RunTest(GetRgba(GetFilePath("input.jpg")),
+          GetRgb(GetFilePath("noop_except_range.png")),
           /*float_ranges=*/{{0.0f, 1.0f}},
           /*int_ranges=*/{{0, 255}, {-128, 127}},
           /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,

mediapipe/calculators/tensor/image_to_tensor_converter.h

@@ -27,12 +27,6 @@ struct Size {
   int height;
 };
 
-// Pixel extrapolation method.
-// When converting image to tensor it may happen that tensor needs to read
-// pixels outside image boundaries. Border mode helps to specify how such pixels
-// will be calculated.
-enum class BorderMode { kZero, kReplicate };
-
 // Converts image to tensor.
 class ImageToTensorConverter {
  public:
@@ -42,13 +36,16 @@ class ImageToTensorConverter {
   // @image contains image to extract from.
   // @roi describes region of interest within the image to extract (absolute
   // values).
-  // @output_dims dimensions of output tensor.
   // @range_min/max describes output tensor range image pixels should converted
   // to.
-  virtual absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
-                                         const RotatedRect& roi,
-                                         const Size& output_dims,
-                                         float range_min, float range_max) = 0;
+  // @tensor_buffer_offset an inteter representing the offset of the tensor
+  // buffer the result should be written to.
+  // @output_tensor a tensor with pre-defined shape. The "Convert" is
+  // responsible of populating the content into the output tensor.
+  virtual absl::Status Convert(const mediapipe::Image& input,
+                               const RotatedRect& roi, float range_min,
+                               float range_max, int tensor_buffer_offset,
+                               Tensor& output_tensor) = 0;
 };
 
 }  // namespace mediapipe

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -264,57 +264,58 @@ class GlProcessor : public ImageToTensorConverter {
     });
   }
 
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
-                                 const RotatedRect& roi,
-                                 const Size& output_dims, float range_min,
-                                 float range_max) override {
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
+                       float range_min, float range_max,
+                       int tensor_buffer_offset,
+                       Tensor& output_tensor) override {
     if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
         input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
-        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGB24) {
       return InvalidArgumentError(absl::StrCat(
-          "Only 4-channel texture input formats are supported, passed format: ",
-          static_cast<uint32_t>(input.format())));
+          "Unsupported format: ", static_cast<uint32_t>(input.format())));
     }
+    const auto& output_shape = output_tensor.shape();
+    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
 
-    constexpr int kNumChannels = 3;
-    Tensor tensor(Tensor::ElementType::kFloat32,
-                  {1, output_dims.height, output_dims.width, kNumChannels});
+    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
+        [this, &output_tensor, &input, &roi, &output_shape, range_min,
+         range_max, tensor_buffer_offset]() -> absl::Status {
+          const int input_num_channels = input.channels();
+          auto source_texture = gl_helper_.CreateSourceTexture(input);
+          tflite::gpu::gl::GlTexture input_texture(
+              GL_TEXTURE_2D, source_texture.name(),
+              input_num_channels == 4 ? GL_RGBA : GL_RGB,
+              source_texture.width() * source_texture.height() *
+                  input_num_channels * sizeof(uint8_t),
+              /*layer=*/0,
+              /*owned=*/false);
 
-    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext([this, &tensor, &input, &roi,
-                                                  &output_dims, range_min,
-                                                  range_max]() -> absl::Status {
-      constexpr int kRgbaNumChannels = 4;
-      auto source_texture = gl_helper_.CreateSourceTexture(input);
-      tflite::gpu::gl::GlTexture input_texture(
-          GL_TEXTURE_2D, source_texture.name(), GL_RGBA,
-          source_texture.width() * source_texture.height() * kRgbaNumChannels *
-              sizeof(uint8_t),
-          /*layer=*/0,
-          /*owned=*/false);
+          constexpr float kInputImageRangeMin = 0.0f;
+          constexpr float kInputImageRangeMax = 1.0f;
+          ASSIGN_OR_RETURN(auto transform,
+                           GetValueRangeTransformation(kInputImageRangeMin,
+                                                       kInputImageRangeMax,
+                                                       range_min, range_max));
 
-      constexpr float kInputImageRangeMin = 0.0f;
-      constexpr float kInputImageRangeMax = 1.0f;
-      ASSIGN_OR_RETURN(
-          auto transform,
-          GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
-                                      range_min, range_max));
+          const int output_size = output_tensor.bytes() / output_shape.dims[0];
+          auto buffer_view = output_tensor.GetOpenGlBufferWriteView();
+          tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER,
+                                           buffer_view.name(), output_size,
+                                           /*offset=*/tensor_buffer_offset,
+                                           /*has_ownership=*/false);
+          MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer(
+              input_texture,
+              tflite::gpu::HW(source_texture.height(), source_texture.width()),
+              roi,
+              /*flip_horizontaly=*/false, transform.scale, transform.offset,
+              tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
+              command_queue_.get(), &output));
 
-      auto buffer_view = tensor.GetOpenGlBufferWriteView();
-      tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER,
-                                       buffer_view.name(), tensor.bytes(),
-                                       /*offset=*/0,
-                                       /*has_ownership=*/false);
-      MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer(
-          input_texture,
-          tflite::gpu::HW(source_texture.height(), source_texture.width()), roi,
-          /*flip_horizontaly=*/false, transform.scale, transform.offset,
-          tflite::gpu::HW(output_dims.height, output_dims.width),
-          command_queue_.get(), &output));
+          return absl::OkStatus();
+        }));
 
-      return absl::OkStatus();
-    }));
-
-    return tensor;
+    return absl::OkStatus();
   }
 
   ~GlProcessor() override {
@@ -326,6 +327,16 @@ class GlProcessor : public ImageToTensorConverter {
   }
 
  private:
+  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
+    RET_CHECK_EQ(output_shape.dims.size(), 4)
+        << "Wrong output dims size: " << output_shape.dims.size();
+    RET_CHECK_GE(output_shape.dims[0], 1)
+        << "The batch dimension needs to be greater or equal to 1.";
+    RET_CHECK_EQ(output_shape.dims[3], 3)
+        << "Wrong output channel: " << output_shape.dims[3];
+    return absl::OkStatus();
+  }
+
   std::unique_ptr<tflite::gpu::gl::CommandQueue> command_queue_;
   std::unique_ptr<SubRectExtractorGl> extractor_;
   mediapipe::GlCalculatorHelper gl_helper_;

mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc

@@ -68,8 +68,8 @@ class GlProcessor : public ImageToTensorConverter {
 
       constexpr GLchar kExtractSubRectVertexShader[] = R"(
             in vec4 position;
-            in mediump vec4 texture_coordinate;
-            out mediump vec2 sample_coordinate;
+            in highp vec4 texture_coordinate;
+            out highp vec2 sample_coordinate;
             uniform mat4 transform_matrix;
 
             void main() {
@@ -86,7 +86,7 @@ class GlProcessor : public ImageToTensorConverter {
           )";
 
       constexpr GLchar kExtractSubRectFragBody[] = R"(
-            DEFAULT_PRECISION(mediump, float)
+            DEFAULT_PRECISION(highp, float)
 
             // Provided by kExtractSubRectVertexShader.
             in vec2 sample_coordinate;
@@ -168,26 +168,26 @@ class GlProcessor : public ImageToTensorConverter {
     });
   }
 
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
-                                 const RotatedRect& roi,
-                                 const Size& output_dims, float range_min,
-                                 float range_max) override {
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
+                       float range_min, float range_max,
+                       int tensor_buffer_offset,
+                       Tensor& output_tensor) override {
     if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
         input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
-        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128 &&
+        input.format() != mediapipe::GpuBufferFormat::kRGB24) {
       return InvalidArgumentError(absl::StrCat(
-          "Only 4-channel texture input formats are supported, passed format: ",
-          static_cast<uint32_t>(input.format())));
+          "Unsupported format: ", static_cast<uint32_t>(input.format())));
     }
+    // TODO: support tensor_buffer_offset > 0 scenario.
+    RET_CHECK_EQ(tensor_buffer_offset, 0)
+        << "The non-zero tensor_buffer_offset input is not supported yet.";
+    const auto& output_shape = output_tensor.shape();
+    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
 
-    constexpr int kNumChannels = 3;
-    Tensor tensor(
-        Tensor::ElementType::kFloat32,
-        Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
-
-    MP_RETURN_IF_ERROR(
-        gl_helper_.RunInGlContext([this, &tensor, &input, &roi, &output_dims,
-                                   range_min, range_max]() -> absl::Status {
+    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
+        [this, &output_tensor, &input, &roi, &output_shape, range_min,
+         range_max]() -> absl::Status {
           auto input_texture = gl_helper_.CreateSourceTexture(input);
 
           constexpr float kInputImageRangeMin = 0.0f;
@@ -196,27 +196,29 @@ class GlProcessor : public ImageToTensorConverter {
                            GetValueRangeTransformation(kInputImageRangeMin,
                                                        kInputImageRangeMax,
                                                        range_min, range_max));
-          auto tensor_view = tensor.GetOpenGlTexture2dWriteView();
+          auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
           MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
                                             /*flip_horizontaly=*/false,
                                             transform.scale, transform.offset,
-                                            output_dims, &tensor_view));
+                                            output_shape, &tensor_view));
           return absl::OkStatus();
         }));
 
-    return tensor;
+    return absl::OkStatus();
   }
 
   absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
                               const RotatedRect& sub_rect,
                               bool flip_horizontaly, float alpha, float beta,
-                              const Size& output_dims,
+                              const Tensor::Shape& output_shape,
                               Tensor::OpenGlTexture2dView* output) {
+    const int output_height = output_shape.dims[1];
+    const int output_width = output_shape.dims[2];
     std::array<float, 16> transform_mat;
 
     glDisable(GL_DEPTH_TEST);
     glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
-    glViewport(0, 0, output_dims.width, output_dims.height);
+    glViewport(0, 0, output_width, output_height);
 
     glActiveTexture(GL_TEXTURE0);
     glBindTexture(GL_TEXTURE_2D, output->name());
@@ -316,6 +318,17 @@ class GlProcessor : public ImageToTensorConverter {
   }
 
  private:
+  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
+    RET_CHECK_EQ(output_shape.dims.size(), 4)
+        << "Wrong output dims size: " << output_shape.dims.size();
+    RET_CHECK_EQ(output_shape.dims[0], 1)
+        << "Handling batch dimension not equal to 1 is not implemented in this "
+           "converter.";
+    RET_CHECK_EQ(output_shape.dims[3], 3)
+        << "Wrong output channel: " << output_shape.dims[3];
+    return absl::OkStatus();
+  }
+
   mediapipe::GlCalculatorHelper gl_helper_;
   bool use_custom_zero_border_ = false;
   BorderMode border_mode_ = BorderMode::kReplicate;

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -36,6 +36,10 @@
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/common/types.h"
 
+#if MEDIAPIPE_METAL_ENABLED
+#include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
+#endif  // MEDIAPIPE_METAL_ENABLED
+
 namespace mediapipe {
 
 namespace {
@@ -262,7 +266,6 @@ class SubRectExtractorMetal {
     RET_CHECK(pipeline_state != nil);
 
     std::string output_type_def;
-    MTLPixelFormat pixel_format;
     switch (output_format) {
       case OutputFormat::kF16C4:
         output_type_def = R"(
@@ -348,10 +351,10 @@ class MetalProcessor : public ImageToTensorConverter {
     return absl::OkStatus();
   }
 
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
-                                 const RotatedRect& roi,
-                                 const Size& output_dims, float range_min,
-                                 float range_max) override {
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
+                       float range_min, float range_max,
+                       int tensor_buffer_offset,
+                       Tensor& output_tensor) override {
     if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
         input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
         input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
@@ -359,16 +362,15 @@ class MetalProcessor : public ImageToTensorConverter {
           "Only 4-channel texture input formats are supported, passed format: ",
           static_cast<uint32_t>(input.format())));
     }
+    RET_CHECK_EQ(tensor_buffer_offset, 0)
+        << "The non-zero tensor_buffer_offset input is not supported yet.";
+    const auto& output_shape = output_tensor.shape();
+    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
 
     @autoreleasepool {
       id<MTLTexture> texture =
           [metal_helper_ metalTextureWithGpuBuffer:input.GetGpuBuffer()];
 
-      constexpr int kNumChannels = 4;
-      Tensor tensor(Tensor::ElementType::kFloat32,
-                    Tensor::Shape{1, output_dims.height, output_dims.width,
-                                  kNumChannels});
-
       constexpr float kInputImageRangeMin = 0.0f;
       constexpr float kInputImageRangeMax = 1.0f;
       ASSIGN_OR_RETURN(
@@ -377,18 +379,30 @@ class MetalProcessor : public ImageToTensorConverter {
                                       range_min, range_max));
 
       id<MTLCommandBuffer> command_buffer = [metal_helper_ commandBuffer];
-      const auto& buffer_view = tensor.GetMtlBufferWriteView(command_buffer);
+      const auto& buffer_view =
+          MtlBufferView::GetWriteView(output_tensor, command_buffer);
       MP_RETURN_IF_ERROR(extractor_->Execute(
           texture, roi,
           /*flip_horizontaly=*/false, transform.scale, transform.offset,
-          tflite::gpu::HW(output_dims.height, output_dims.width),
+          tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
           command_buffer, buffer_view.buffer()));
       [command_buffer commit];
-      return tensor;
+      return absl::OkStatus();
     }
   }
 
  private:
+  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
+    RET_CHECK_EQ(output_shape.dims.size(), 4)
+        << "Wrong output dims size: " << output_shape.dims.size();
+    RET_CHECK_EQ(output_shape.dims[0], 1)
+        << "Handling batch dimension not equal to 1 is not implemented in this "
+           "converter.";
+    RET_CHECK_EQ(output_shape.dims[3], 4)
+        << "Wrong output channel: " << output_shape.dims[3];
+    return absl::OkStatus();
+  }
+
   MPPMetalHelper* metal_helper_ = nil;
   std::unique_ptr<SubRectExtractorMetal> extractor_;
 };

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc

@@ -48,47 +48,77 @@ class OpenCvProcessor : public ImageToTensorConverter {
     switch (tensor_type_) {
       case Tensor::ElementType::kInt8:
         mat_type_ = CV_8SC3;
+        mat_gray_type_ = CV_8SC1;
         break;
       case Tensor::ElementType::kFloat32:
         mat_type_ = CV_32FC3;
+        mat_gray_type_ = CV_32FC1;
         break;
       case Tensor::ElementType::kUInt8:
         mat_type_ = CV_8UC3;
+        mat_gray_type_ = CV_8UC1;
         break;
       default:
         mat_type_ = -1;
+        mat_gray_type_ = -1;
     }
   }
 
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
-                                 const RotatedRect& roi,
-                                 const Size& output_dims, float range_min,
-                                 float range_max) override {
-    if (input.image_format() != mediapipe::ImageFormat::SRGB &&
-        input.image_format() != mediapipe::ImageFormat::SRGBA) {
-      return InvalidArgumentError(
-          absl::StrCat("Only RGBA/RGB formats are supported, passed format: ",
-                       static_cast<uint32_t>(input.image_format())));
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
+                       float range_min, float range_max,
+                       int tensor_buffer_offset,
+                       Tensor& output_tensor) override {
+    const bool is_supported_format =
+        input.image_format() == mediapipe::ImageFormat::SRGB ||
+        input.image_format() == mediapipe::ImageFormat::SRGBA ||
+        input.image_format() == mediapipe::ImageFormat::GRAY8;
+    if (!is_supported_format) {
+      return InvalidArgumentError(absl::StrCat(
+          "Unsupported format: ", static_cast<uint32_t>(input.image_format())));
     }
-    auto src = mediapipe::formats::MatView(&input);
 
-    constexpr int kNumChannels = 3;
-    Tensor tensor(tensor_type_, Tensor::Shape{1, output_dims.height,
-                                              output_dims.width, kNumChannels});
-    auto buffer_view = tensor.GetCpuWriteView();
+    RET_CHECK_GE(tensor_buffer_offset, 0)
+        << "The input tensor_buffer_offset needs to be non-negative.";
+    const auto& output_shape = output_tensor.shape();
+    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
+
+    const int output_height = output_shape.dims[1];
+    const int output_width = output_shape.dims[2];
+    const int output_channels = output_shape.dims[3];
+    const int num_elements_per_img =
+        output_height * output_width * output_channels;
+    auto buffer_view = output_tensor.GetCpuWriteView();
     cv::Mat dst;
+    const int dst_data_type = output_channels == 1 ? mat_gray_type_ : mat_type_;
     switch (tensor_type_) {
       case Tensor::ElementType::kInt8:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
-                      buffer_view.buffer<int8>());
+        RET_CHECK_GE(output_shape.num_elements(),
+                     tensor_buffer_offset / sizeof(int8) + num_elements_per_img)
+            << "The buffer offset + the input image size is larger than the "
+               "allocated tensor buffer.";
+        dst = cv::Mat(
+            output_height, output_width, dst_data_type,
+            buffer_view.buffer<int8>() + tensor_buffer_offset / sizeof(int8));
         break;
       case Tensor::ElementType::kFloat32:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
-                      buffer_view.buffer<float>());
+        RET_CHECK_GE(
+            output_shape.num_elements(),
+            tensor_buffer_offset / sizeof(float) + num_elements_per_img)
+            << "The buffer offset + the input image size is larger than the "
+               "allocated tensor buffer.";
+        dst = cv::Mat(
+            output_height, output_width, dst_data_type,
+            buffer_view.buffer<float>() + tensor_buffer_offset / sizeof(float));
         break;
       case Tensor::ElementType::kUInt8:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
-                      buffer_view.buffer<uint8>());
+        RET_CHECK_GE(
+            output_shape.num_elements(),
+            tensor_buffer_offset / sizeof(uint8) + num_elements_per_img)
+            << "The buffer offset + the input image size is larger than the "
+               "allocated tensor buffer.";
+        dst = cv::Mat(
+            output_height, output_width, dst_data_type,
+            buffer_view.buffer<uint8>() + tensor_buffer_offset / sizeof(uint8));
         break;
       default:
         return InvalidArgumentError(
@@ -101,8 +131,8 @@ class OpenCvProcessor : public ImageToTensorConverter {
     cv::Mat src_points;
     cv::boxPoints(rotated_rect, src_points);
 
-    const float dst_width = output_dims.width;
-    const float dst_height = output_dims.height;
+    const float dst_width = output_width;
+    const float dst_height = output_height;
     /* clang-format off */
     float dst_corners[8] = {0.0f,      dst_height,
                             0.0f,      0.0f,
@@ -110,6 +140,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
                             dst_width, dst_height};
     /* clang-format on */
 
+    auto src = mediapipe::formats::MatView(&input);
     cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
     cv::Mat projection_matrix =
         cv::getPerspectiveTransform(src_points, dst_points);
@@ -119,7 +150,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
                         /*flags=*/cv::INTER_LINEAR,
                         /*borderMode=*/border_mode_);
 
-    if (transformed.channels() > kNumChannels) {
+    if (transformed.channels() > output_channels) {
       cv::Mat proper_channels_mat;
       cv::cvtColor(transformed, proper_channels_mat, cv::COLOR_RGBA2RGB);
       transformed = proper_channels_mat;
@@ -131,14 +162,26 @@ class OpenCvProcessor : public ImageToTensorConverter {
         auto transform,
         GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                     range_min, range_max));
-    transformed.convertTo(dst, mat_type_, transform.scale, transform.offset);
-    return tensor;
+    transformed.convertTo(dst, dst_data_type, transform.scale,
+                          transform.offset);
+    return absl::OkStatus();
   }
 
  private:
+  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
+    RET_CHECK_EQ(output_shape.dims.size(), 4)
+        << "Wrong output dims size: " << output_shape.dims.size();
+    RET_CHECK_GE(output_shape.dims[0], 1)
+        << "The batch dimension needs to be equal or larger than 1.";
+    RET_CHECK(output_shape.dims[3] == 3 || output_shape.dims[3] == 1)
+        << "Wrong output channel: " << output_shape.dims[3];
+    return absl::OkStatus();
+  }
+
   enum cv::BorderTypes border_mode_;
   Tensor::ElementType tensor_type_;
   int mat_type_;
+  int mat_gray_type_;
 };
 
 }  // namespace

mediapipe/calculators/tensor/image_to_tensor_utils.cc

@@ -16,7 +16,9 @@
 
 #include <array>
 
+#include "absl/status/status.h"
 #include "absl/types/optional.h"
+#include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/statusor.h"
 
@@ -214,4 +216,75 @@ void GetTransposedRotatedSubRectToRectTransformMatrix(
   matrix[15] = 1.0f;
 }
 
+BorderMode GetBorderMode(
+    const mediapipe::ImageToTensorCalculatorOptions::BorderMode& mode) {
+  switch (mode) {
+    case mediapipe::
+        ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED:
+      return BorderMode::kReplicate;
+    case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO:
+      return BorderMode::kZero;
+    case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE:
+      return BorderMode::kReplicate;
+  }
+}
+
+Tensor::ElementType GetOutputTensorType(bool uses_gpu,
+                                        const OutputTensorParams& params) {
+  if (!uses_gpu) {
+    if (params.is_float_output) {
+      return Tensor::ElementType::kFloat32;
+    }
+    if (params.range_min < 0) {
+      return Tensor::ElementType::kInt8;
+    } else {
+      return Tensor::ElementType::kUInt8;
+    }
+  }
+  // Always use float32 when GPU is enabled.
+  return Tensor::ElementType::kFloat32;
+}
+
+int GetNumOutputChannels(const mediapipe::Image& image) {
+#if !MEDIAPIPE_DISABLE_GPU
+#if MEDIAPIPE_METAL_ENABLED
+  if (image.UsesGpu()) {
+    return 4;
+  }
+#endif  // MEDIAPIPE_METAL_ENABLED
+#endif  // !MEDIAPIPE_DISABLE_GPU
+  // TODO: Add a unittest here to test the behavior on GPU, i.e.
+  // failure.
+  // Only output channel == 1 when running on CPU and the input image channel
+  // is 1. Ideally, we want to also support GPU for output channel == 1. But
+  // setting this on the safer side to prevent unintentional failure.
+  if (!image.UsesGpu() && image.channels() == 1) {
+    return 1;
+  }
+  return 3;
+}
+
+absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
+    const api2::Packet<api2::OneOf<Image, mediapipe::ImageFrame>>&
+        image_packet) {
+  return image_packet.Visit(
+      [&image_packet](const mediapipe::Image&) {
+        return SharedPtrWithPacket<mediapipe::Image>(image_packet);
+      },
+      [&image_packet](const mediapipe::ImageFrame&) {
+        return std::make_shared<const mediapipe::Image>(
+            std::const_pointer_cast<mediapipe::ImageFrame>(
+                SharedPtrWithPacket<mediapipe::ImageFrame>(image_packet)));
+      });
+}
+
+#if !MEDIAPIPE_DISABLE_GPU
+absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
+    const api2::Packet<mediapipe::GpuBuffer>& image_gpu_packet) {
+  // A shallow copy is okay since the resulting 'image' object is local in
+  // Process(), and thus never outlives 'input'.
+  return std::make_shared<const mediapipe::Image>(image_gpu_packet.Get());
+}
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
 }  // namespace mediapipe

mediapipe/calculators/tensor/image_to_tensor_utils.h

@@ -18,8 +18,18 @@
 #include <array>
 
 #include "absl/types/optional.h"
+#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/statusor.h"
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gpu_buffer.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gpu_origin.pb.h"
 
 namespace mediapipe {
 
@@ -31,6 +41,24 @@ struct RotatedRect {
   float rotation;
 };
 
+// Pixel extrapolation method.
+// When converting image to tensor it may happen that tensor needs to read
+// pixels outside image boundaries. Border mode helps to specify how such pixels
+// will be calculated.
+// TODO: Consider moving this to a separate border_mode.h file.
+enum class BorderMode { kZero, kReplicate };
+
+// Struct that host commonly accessed parameters used in the
+// ImageTo[Batch]TensorCalculator.
+struct OutputTensorParams {
+  int output_height;
+  int output_width;
+  int output_batch;
+  bool is_float_output;
+  float range_min;
+  float range_max;
+};
+
 // Generates a new ROI or converts it from normalized rect.
 RotatedRect GetRoi(int input_width, int input_height,
                    absl::optional<mediapipe::NormalizedRect> norm_rect);
@@ -95,6 +123,103 @@ void GetTransposedRotatedSubRectToRectTransformMatrix(
     const RotatedRect& sub_rect, int rect_width, int rect_height,
     bool flip_horizontaly, std::array<float, 16>* matrix);
 
+// Validates the output dimensions set in the option proto. The input option
+// proto is expected to have to following fields:
+//  output_tensor_float_range, output_tensor_int_range, output_tensor_uint_range
+//  output_tensor_width, output_tensor_height.
+// See ImageToTensorCalculatorOptions for the description of each field.
+template <typename T>
+absl::Status ValidateOptionOutputDims(const T& options) {
+  RET_CHECK(options.has_output_tensor_float_range() ||
+            options.has_output_tensor_int_range() ||
+            options.has_output_tensor_uint_range())
+      << "Output tensor range is required.";
+  if (options.has_output_tensor_float_range()) {
+    RET_CHECK_LT(options.output_tensor_float_range().min(),
+                 options.output_tensor_float_range().max())
+        << "Valid output float tensor range is required.";
+  }
+  if (options.has_output_tensor_uint_range()) {
+    RET_CHECK_LT(options.output_tensor_uint_range().min(),
+                 options.output_tensor_uint_range().max())
+        << "Valid output uint tensor range is required.";
+    RET_CHECK_GE(options.output_tensor_uint_range().min(), 0)
+        << "The minimum of the output uint tensor range must be "
+           "non-negative.";
+    RET_CHECK_LE(options.output_tensor_uint_range().max(), 255)
+        << "The maximum of the output uint tensor range must be less than or "
+           "equal to 255.";
+  }
+  if (options.has_output_tensor_int_range()) {
+    RET_CHECK_LT(options.output_tensor_int_range().min(),
+                 options.output_tensor_int_range().max())
+        << "Valid output int tensor range is required.";
+    RET_CHECK_GE(options.output_tensor_int_range().min(), -128)
+        << "The minimum of the output int tensor range must be greater than "
+           "or equal to -128.";
+    RET_CHECK_LE(options.output_tensor_int_range().max(), 127)
+        << "The maximum of the output int tensor range must be less than or "
+           "equal to 127.";
+  }
+  RET_CHECK_GT(options.output_tensor_width(), 0)
+      << "Valid output tensor width is required.";
+  RET_CHECK_GT(options.output_tensor_height(), 0)
+      << "Valid output tensor height is required.";
+  return absl::OkStatus();
+}
+
+template <typename T>
+OutputTensorParams GetOutputTensorParams(const T& options) {
+  OutputTensorParams params;
+  if (options.has_output_tensor_uint_range()) {
+    params.range_min =
+        static_cast<float>(options.output_tensor_uint_range().min());
+    params.range_max =
+        static_cast<float>(options.output_tensor_uint_range().max());
+  } else if (options.has_output_tensor_int_range()) {
+    params.range_min =
+        static_cast<float>(options.output_tensor_int_range().min());
+    params.range_max =
+        static_cast<float>(options.output_tensor_int_range().max());
+  } else {
+    params.range_min = options.output_tensor_float_range().min();
+    params.range_max = options.output_tensor_float_range().max();
+  }
+  params.output_width = options.output_tensor_width();
+  params.output_height = options.output_tensor_height();
+  params.is_float_output = options.has_output_tensor_float_range();
+  params.output_batch = 1;
+  return params;
+}
+
+// Returns whether the GPU input format starts at the bottom.
+template <typename T>
+bool DoesGpuInputStartAtBottom(const T& options) {
+  return options.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
+}
+
+// Converts the BorderMode proto into struct.
+BorderMode GetBorderMode(
+    const mediapipe::ImageToTensorCalculatorOptions::BorderMode& mode);
+
+// Gets the output tensor type.
+Tensor::ElementType GetOutputTensorType(bool uses_gpu,
+                                        const OutputTensorParams& params);
+
+// Gets the number of output channels from the input Image format.
+int GetNumOutputChannels(const mediapipe::Image& image);
+
+// Converts the packet that hosts different format (Image, ImageFrame,
+// GpuBuffer) into the mediapipe::Image format.
+absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
+    const api2::Packet<api2::OneOf<Image, mediapipe::ImageFrame>>&
+        image_packet);
+
+#if !MEDIAPIPE_DISABLE_GPU
+absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
+    const api2::Packet<mediapipe::GpuBuffer>& image_gpu_packet);
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_

mediapipe/calculators/tensor/image_to_tensor_utils_test.cc

@@ -16,6 +16,8 @@
 
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_macros.h"
 #include "mediapipe/framework/port/status_matchers.h"
 
 namespace mediapipe {
@@ -23,6 +25,7 @@ namespace {
 
 using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
+using ::testing::HasSubstr;
 
 testing::Matcher<RotatedRect> EqRotatedRect(float width, float height,
                                             float center_x, float center_y,
@@ -157,5 +160,95 @@ TEST(GetValueRangeTransformation, FloatToPixel) {
               EqValueTransformation(/*scale=*/255.0f, /*offset=*/0.0f));
 }
 
+constexpr char kValidFloatProto[] = R"(
+  output_tensor_float_range { min: 0.0 max: 1.0 }
+  output_tensor_width: 100
+  output_tensor_height: 200
+)";
+
+constexpr char kValidIntProto[] = R"(
+  output_tensor_float_range { min: 0 max: 255 }
+  output_tensor_width: 100
+  output_tensor_height: 200
+)";
+
+TEST(ValidateOptionOutputDims, ImageToTensorCalcOptions) {
+  const auto float_options =
+      mediapipe::ParseTextProtoOrDie<mediapipe::ImageToTensorCalculatorOptions>(
+          kValidFloatProto);
+  MP_EXPECT_OK(ValidateOptionOutputDims(float_options));
+}
+
+TEST(ValidateOptionOutputDims, EmptyProto) {
+  mediapipe::ImageToTensorCalculatorOptions options;
+  // No output tensor range set.
+  EXPECT_THAT(ValidateOptionOutputDims(options),
+              StatusIs(absl::StatusCode::kInternal,
+                       HasSubstr("Output tensor range is required")));
+
+  // Invalid output float tensor range.
+  options.mutable_output_tensor_float_range()->set_min(1.0);
+  options.mutable_output_tensor_float_range()->set_max(0.0);
+  EXPECT_THAT(
+      ValidateOptionOutputDims(options),
+      StatusIs(absl::StatusCode::kInternal,
+               HasSubstr("Valid output float tensor range is required")));
+
+  // Output width/height is not set.
+  options.mutable_output_tensor_float_range()->set_min(0.0);
+  options.mutable_output_tensor_float_range()->set_max(1.0);
+  EXPECT_THAT(ValidateOptionOutputDims(options),
+              StatusIs(absl::StatusCode::kInternal,
+                       HasSubstr("Valid output tensor width is required")));
+}
+
+TEST(GetOutputTensorParams, ImageToTensorCalcOptionsSetValues) {
+  // Test int range with ImageToTensorCalculatorOptions.
+  const auto int_options =
+      mediapipe::ParseTextProtoOrDie<mediapipe::ImageToTensorCalculatorOptions>(
+          kValidIntProto);
+  const auto params2 = GetOutputTensorParams(int_options);
+  EXPECT_EQ(params2.range_min, 0.0f);
+  EXPECT_EQ(params2.range_max, 255.0f);
+  EXPECT_EQ(params2.output_batch, 1);
+  EXPECT_EQ(params2.output_width, 100);
+  EXPECT_EQ(params2.output_height, 200);
+}
+
+TEST(GetBorderMode, GetBorderMode) {
+  // Default to REPLICATE.
+  auto border_mode =
+      mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED;
+  EXPECT_EQ(BorderMode::kReplicate, GetBorderMode(border_mode));
+
+  // Set to ZERO.
+  border_mode =
+      mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO;
+  EXPECT_EQ(BorderMode::kZero, GetBorderMode(border_mode));
+}
+
+TEST(GetOutputTensorType, GetOutputTensorType) {
+  OutputTensorParams params;
+  // Return float32 when GPU is enabled.
+  EXPECT_EQ(Tensor::ElementType::kFloat32,
+            GetOutputTensorType(/*uses_gpu=*/true, params));
+
+  // Return float32 when is_float_output is set to true.
+  params.is_float_output = true;
+  EXPECT_EQ(Tensor::ElementType::kFloat32,
+            GetOutputTensorType(/*uses_gpu=*/false, params));
+
+  // Return int8 when range_min is negative.
+  params.is_float_output = false;
+  params.range_min = -255.0f;
+  EXPECT_EQ(Tensor::ElementType::kInt8,
+            GetOutputTensorType(/*uses_gpu=*/false, params));
+
+  // Return 8int8 when range_min is non-negative.
+  params.range_min = 0.0f;
+  EXPECT_EQ(Tensor::ElementType::kUInt8,
+            GetOutputTensorType(/*uses_gpu=*/false, params));
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensor/inference_calculator.proto

@@ -17,6 +17,7 @@ syntax = "proto2";
 package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
+import "mediapipe/framework/calculator_options.proto";
 
 option java_package = "com.google.mediapipe.calculator.proto";
 option java_outer_classname = "InferenceCalculatorProto";

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -72,7 +72,7 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
   RET_CHECK(!input_tensors.empty());
 
   ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
-                   inference_runner_->Run(input_tensors));
+                   inference_runner_->Run(cc, input_tensors));
   kOutTensors(cc).Send(std::move(output_tensors));
   return absl::OkStatus();
 }
@@ -151,6 +151,8 @@ InferenceCalculatorCpuImpl::MaybeCreateDelegate(CalculatorContext* cc) {
     auto xnnpack_opts = TfLiteXNNPackDelegateOptionsDefault();
     xnnpack_opts.num_threads =
         GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
+    // TODO Remove once XNNPACK is enabled by default.
+    xnnpack_opts.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
     return TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
                              &TfLiteXNNPackDelegateDelete);
   }

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -20,12 +20,15 @@
 
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
+#include "absl/strings/str_format.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
 #include "mediapipe/calculators/tensor/inference_calculator.pb.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
 
+#define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
+
 namespace mediapipe {
 namespace api2 {
 
@@ -152,6 +155,10 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
   const auto& input_indices = interpreter_->inputs();
   for (int i = 0; i < input_indices.size(); ++i) {
     const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
+    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
+        "Input tensor at index [%d] doesn't specify dimensions.",
+        input_indices[i]);
+
     gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
         Tensor::ElementType::kFloat32,
         Tensor::Shape{std::vector<int>{
@@ -169,6 +176,9 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
   // Create and bind output buffers.
   for (int i = 0; i < output_size_; ++i) {
     const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
+    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
+        "Output tensor at index [%d] doesn't specify dimensions.",
+        output_indices[i]);
     gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
         Tensor::ElementType::kFloat32,
         Tensor::Shape{std::vector<int>{
@@ -191,7 +201,7 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
     CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
     std::vector<Tensor>& output_tensors) {
   return gpu_helper_.RunInGlContext(
-      [this, &input_tensors, &output_tensors]() -> absl::Status {
+      [this, cc, &input_tensors, &output_tensors]() -> absl::Status {
         // Explicitly copy input.
         for (int i = 0; i < input_tensors.size(); ++i) {
           glBindBuffer(GL_COPY_READ_BUFFER,
@@ -203,7 +213,10 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
         }
 
         // Run inference.
-        RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+        {
+          MEDIAPIPE_PROFILING(GPU_TASK_INVOKE, cc);
+          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+        }
 
         output_tensors.reserve(output_size_);
         for (int i = 0; i < output_size_; ++i) {

mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc

@@ -25,12 +25,14 @@
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/util/tflite/tflite_gpu_runner.h"
 
-#if defined(MEDIAPIPE_ANDROID)
+#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/filesystem.h"
 #include "mediapipe/util/android/file/base/helpers.h"
-#endif  // MEDIAPIPE_ANDROID
+#endif  // defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
+
+#define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
 
 namespace mediapipe {
 namespace api2 {
@@ -83,7 +85,7 @@ class InferenceCalculatorGlAdvancedImpl
         const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
 
     absl::StatusOr<std::vector<Tensor>> Process(
-        const std::vector<Tensor>& input_tensors);
+        CalculatorContext* cc, const std::vector<Tensor>& input_tensors);
 
     absl::Status Close();
 
@@ -121,11 +123,11 @@ absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init(
 
 absl::StatusOr<std::vector<Tensor>>
 InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
-    const std::vector<Tensor>& input_tensors) {
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
   std::vector<Tensor> output_tensors;
 
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-      [this, &input_tensors, &output_tensors]() -> absl::Status {
+      [this, cc, &input_tensors, &output_tensors]() -> absl::Status {
         for (int i = 0; i < input_tensors.size(); ++i) {
           MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
               input_tensors[i].GetOpenGlBufferReadView().name(), i));
@@ -138,7 +140,10 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
               output_tensors.back().GetOpenGlBufferWriteView().name(), i));
         }
         // Run inference.
-        return tflite_gpu_runner_->Invoke();
+        {
+          MEDIAPIPE_PROFILING(GPU_TASK_INVOKE, cc);
+          return tflite_gpu_runner_->Invoke();
+        }
       }));
 
   return output_tensors;
@@ -226,19 +231,26 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
   return tflite_gpu_runner_->Build();
 }
 
-#if defined(MEDIAPIPE_ANDROID)
+#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
     const mediapipe::InferenceCalculatorOptions& options,
     const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
         gpu_delegate_options) {
-  use_kernel_caching_ = gpu_delegate_options.has_cached_kernel_path();
+  // The kernel cache needs a unique filename based on either model_path or the
+  // model token, to prevent the cache from being overwritten if the graph has
+  // more than one model.
+  use_kernel_caching_ =
+      gpu_delegate_options.has_cached_kernel_path() &&
+      (options.has_model_path() || gpu_delegate_options.has_model_token());
   use_serialized_model_ = gpu_delegate_options.has_serialized_model_dir() &&
                           gpu_delegate_options.has_model_token();
 
   if (use_kernel_caching_) {
-    cached_kernel_filename_ = gpu_delegate_options.cached_kernel_path() +
-                              mediapipe::File::Basename(options.model_path()) +
-                              ".ker";
+    std::string basename = options.has_model_path()
+                               ? mediapipe::File::Basename(options.model_path())
+                               : gpu_delegate_options.model_token();
+    cached_kernel_filename_ = mediapipe::file::JoinPath(
+        gpu_delegate_options.cached_kernel_path(), basename + ".ker");
   }
   if (use_serialized_model_) {
     serialized_model_path_ =
@@ -253,9 +265,9 @@ InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
     tflite::gpu::TFLiteGPURunner* gpu_runner) const {
   if (use_kernel_caching_) {
     // Save kernel file.
-    auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
-        gpu_runner->GetSerializedBinaryCache());
-    std::string cache_str(kernel_cache->begin(), kernel_cache->end());
+    ASSIGN_OR_RETURN(std::vector<uint8_t> kernel_cache,
+                     gpu_runner->GetSerializedBinaryCache());
+    std::string cache_str(kernel_cache.begin(), kernel_cache.end());
     MP_RETURN_IF_ERROR(
         mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
   }
@@ -313,7 +325,7 @@ InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
     tflite::gpu::TFLiteGPURunner* gpu_runner) const {
   return absl::OkStatus();
 }
-#endif  // MEDIAPIPE_ANDROID
+#endif  // defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 
 absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
     CalculatorContract* cc) {
@@ -354,7 +366,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
   auto output_tensors = absl::make_unique<std::vector<Tensor>>();
 
   ASSIGN_OR_RETURN(*output_tensors,
-                   gpu_inference_runner_->Process(input_tensors));
+                   gpu_inference_runner_->Process(cc, input_tensors));
 
   kOutTensors(cc).Send(std::move(output_tensors));
   return absl::OkStatus();

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -22,7 +22,10 @@
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "absl/strings/str_format.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #include "mediapipe/gpu/gpu_buffer.h"
@@ -149,11 +152,12 @@ absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
   command_buffer.label = @"InferenceCalculator";
   // Explicit copy input with conversion float 32 bits to 16 bits.
   for (int i = 0; i < input_tensors.size(); ++i) {
-    auto input_view = input_tensors[i].GetMtlBufferReadView(command_buffer);
+    auto input_view =
+        MtlBufferView::GetReadView(input_tensors[i], command_buffer);
     // Reshape tensor.
     tflite::gpu::BHWC shape = BhwcFromTensorShape(input_tensors[i].shape());
     auto gpu_buffer_view =
-        gpu_buffers_in_[i]->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*gpu_buffers_in_[i], command_buffer);
     id<MTLComputeCommandEncoder> input_encoder =
         [command_buffer computeCommandEncoder];
     [converter_to_BPHWC4_ convertWithEncoder:input_encoder
@@ -173,9 +177,10 @@ absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
                                  output_shapes_[i]);
     // Reshape tensor.
     tflite::gpu::BHWC shape = BhwcFromTensorShape(output_shapes_[i]);
-    auto read_view = gpu_buffers_out_[i]->GetMtlBufferReadView(command_buffer);
+    auto read_view =
+        MtlBufferView::GetReadView(*gpu_buffers_out_[i], command_buffer);
     auto write_view =
-        output_tensors->at(i).GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(output_tensors->at(i), command_buffer);
     id<MTLComputeCommandEncoder> output_encoder =
         [command_buffer computeCommandEncoder];
     [converter_from_BPHWC4_ convertWithEncoder:output_encoder
@@ -224,9 +229,6 @@ absl::Status InferenceCalculatorMetalImpl::InitInterpreter(
 
 void InferenceCalculatorMetalImpl::AddDelegate(
     CalculatorContext* cc, tflite::InterpreterBuilder* interpreter_builder) {
-  const auto& calculator_opts =
-      cc->Options<mediapipe::InferenceCalculatorOptions>();
-
   // Configure and create the delegate.
   TFLGpuDelegateOptions options;
   // `enable_quantization` enables the run of sparse models i.e. the models with
@@ -248,6 +250,9 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
   const auto& input_indices = interpreter_->inputs();
   for (int i = 0; i < input_indices.size(); ++i) {
     const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
+    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
+        "Input tensor at index [%d] doesn't specify dimensions.",
+        input_indices[i]);
     // Create and bind input buffer.
     std::vector<int> dims{tensor->dims->data,
                           tensor->dims->data + tensor->dims->size};
@@ -257,7 +262,7 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
                               : Tensor::ElementType::kFloat32,
         Tensor::Shape{dims}));
     auto buffer_view =
-        gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice);
+        MtlBufferView::GetWriteView(*gpu_buffers_in_[i], gpu_helper_.mtlDevice);
     RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                      delegate_.get(), input_indices[i], buffer_view.buffer()),
                  true);
@@ -269,6 +274,9 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
   output_shapes_.resize(output_indices.size());
   for (int i = 0; i < output_shapes_.size(); ++i) {
     const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
+    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
+        "Output tensor at index [%d] doesn't specify dimensions.",
+        output_indices[i]);
     RET_CHECK(tensor->dims->size <= 4);
     // Create and bind output buffers.
     // Channels are always padded to multiple of 4.
@@ -282,8 +290,8 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
         Tensor::Shape{dims}));
     RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                      delegate_.get(), output_indices[i],
-                     gpu_buffers_out_[i]
-                         ->GetMtlBufferWriteView(gpu_helper_.mtlDevice)
+                     MtlBufferView::GetWriteView(*gpu_buffers_out_[i],
+                                                 gpu_helper_.mtlDevice)
                          .buffer()),
                  true);
   }

mediapipe/calculators/tensor/inference_calculator_xnnpack.cc

@@ -70,7 +70,7 @@ absl::Status InferenceCalculatorXnnpackImpl::Process(CalculatorContext* cc) {
   RET_CHECK(!input_tensors.empty());
 
   ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
-                   inference_runner_->Run(input_tensors));
+                   inference_runner_->Run(cc, input_tensors));
   kOutTensors(cc).Send(std::move(output_tensors));
   return absl::OkStatus();
 }
@@ -114,6 +114,8 @@ InferenceCalculatorXnnpackImpl::CreateDelegate(CalculatorContext* cc) {
   auto xnnpack_opts = TfLiteXNNPackDelegateOptionsDefault();
   xnnpack_opts.num_threads =
       GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
+  // TODO Remove once XNNPACK is enabled by default.
+  xnnpack_opts.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
   return TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
                            &TfLiteXNNPackDelegateDelete);
 }

mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc

@@ -20,9 +20,14 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/mediapipe_profiling.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "tensorflow/lite/c/c_api_types.h"
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/interpreter_builder.h"
+#include "tensorflow/lite/string_util.h"
+
+#define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
 
 namespace mediapipe {
 
@@ -39,6 +44,19 @@ void CopyTensorBufferToInterpreter(const Tensor& input_tensor,
   std::memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor.bytes());
 }
 
+template <>
+void CopyTensorBufferToInterpreter<char>(const Tensor& input_tensor,
+                                         tflite::Interpreter* interpreter,
+                                         int input_tensor_index) {
+  const char* input_tensor_buffer =
+      input_tensor.GetCpuReadView().buffer<char>();
+  tflite::DynamicBuffer dynamic_buffer;
+  dynamic_buffer.AddString(input_tensor_buffer,
+                           input_tensor.shape().num_elements());
+  dynamic_buffer.WriteToTensorAsVector(
+      interpreter->tensor(interpreter->inputs()[input_tensor_index]));
+}
+
 template <typename T>
 void CopyTensorBufferFromInterpreter(tflite::Interpreter* interpreter,
                                      int output_tensor_index,
@@ -64,7 +82,7 @@ class InferenceInterpreterDelegateRunner : public InferenceRunner {
         delegate_(std::move(delegate)) {}
 
   absl::StatusOr<std::vector<Tensor>> Run(
-      const std::vector<Tensor>& input_tensors) override;
+      CalculatorContext* cc, const std::vector<Tensor>& input_tensors) override;
 
  private:
   api2::Packet<TfLiteModelPtr> model_;
@@ -73,7 +91,7 @@ class InferenceInterpreterDelegateRunner : public InferenceRunner {
 };
 
 absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
-    const std::vector<Tensor>& input_tensors) {
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
   // Read CPU input into tensors.
   RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
   for (int i = 0; i < input_tensors.size(); ++i) {
@@ -87,13 +105,13 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
         break;
       }
       case TfLiteType::kTfLiteUInt8: {
-        CopyTensorBufferToInterpreter<uint8>(input_tensors[i],
-                                             interpreter_.get(), i);
+        CopyTensorBufferToInterpreter<uint8_t>(input_tensors[i],
+                                               interpreter_.get(), i);
         break;
       }
       case TfLiteType::kTfLiteInt8: {
-        CopyTensorBufferToInterpreter<int8>(input_tensors[i],
-                                            interpreter_.get(), i);
+        CopyTensorBufferToInterpreter<int8_t>(input_tensors[i],
+                                              interpreter_.get(), i);
         break;
       }
       case TfLiteType::kTfLiteInt32: {
@@ -101,6 +119,14 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
                                                interpreter_.get(), i);
         break;
       }
+      case TfLiteType::kTfLiteString: {
+        CopyTensorBufferToInterpreter<char>(input_tensors[i],
+                                            interpreter_.get(), i);
+        break;
+      }
+      case TfLiteType::kTfLiteBool:
+        // No current use-case for copying MediaPipe Tensors with bool type to
+        // TfLiteTensors.
       default:
         return absl::InvalidArgumentError(
             absl::StrCat("Unsupported input tensor type:", input_tensor_type));
@@ -108,8 +134,10 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
   }
 
   // Run inference.
-  RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
-
+  {
+    MEDIAPIPE_PROFILING(CPU_TASK_INVOKE, cc);
+    RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+  }
   // Output result tensors (CPU).
   const auto& tensor_indexes = interpreter_->outputs();
   std::vector<Tensor> output_tensors;
@@ -146,6 +174,15 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
         CopyTensorBufferFromInterpreter<int32_t>(interpreter_.get(), i,
                                                  &output_tensors.back());
         break;
+      case TfLiteType::kTfLiteBool:
+        output_tensors.emplace_back(Tensor::ElementType::kBool, shape,
+                                    Tensor::QuantizationParameters{1.0f, 0});
+        CopyTensorBufferFromInterpreter<bool>(interpreter_.get(), i,
+                                              &output_tensors.back());
+        break;
+      case TfLiteType::kTfLiteString:
+        // No current use-case for copying TfLiteTensors with string type to
+        // MediaPipe Tensors.
       default:
         return absl::InvalidArgumentError(
             absl::StrCat("Unsupported output tensor type:",

mediapipe/calculators/tensor/inference_runner.h

@@ -2,6 +2,7 @@
 #define MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_H_
 
 #include "absl/status/statusor.h"
+#include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/formats/tensor.h"
 
 namespace mediapipe {
@@ -11,7 +12,7 @@ class InferenceRunner {
  public:
   virtual ~InferenceRunner() = default;
   virtual absl::StatusOr<std::vector<Tensor>> Run(
-      const std::vector<Tensor>& inputs) = 0;
+      CalculatorContext* cc, const std::vector<Tensor>& inputs) = 0;
 };
 
 }  // namespace mediapipe

mediapipe/calculators/tensor/regex_preprocessor_calculator.cc

@@ -0,0 +1,174 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
+#include "mediapipe/tasks/cc/text/tokenizers/regex_tokenizer.h"
+#include "mediapipe/tasks/cc/text/tokenizers/tokenizer_utils.h"
+#include "mediapipe/tasks/metadata/metadata_schema_generated.h"
+
+namespace mediapipe {
+namespace api2 {
+
+using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
+
+// Preprocesses input text into one int32 input tensor for a text model using
+// a RegexTokenizer.
+//
+// Inputs:
+//   TEXT - std::string
+//     The input text.
+// Side Inputs:
+//   METADATA_EXTRACTOR - ModelMetadataExtractor
+//     The metadata extractor for the text model. Used to extract the metadata
+//     to construct the RegexTokenizer.
+//
+// Outputs:
+//   TENSORS - std::vector<Tensor>
+//     Vector containing a single Tensor which is the text model's input tensor.
+//     Depending on the tokenizer metadata, the tensor may start with
+//     the id of the tokenizer's <START> token. The following tensor values will
+//     be the ids of the tokens of the input text. Any out-of-vocab tokens will
+//     have the id of the <UNKNOWN> token. The tensor will be padded with the
+//     <PAD> token id to have size equal to the max sequence length for the text
+//     model.
+//
+// Example:
+// node {
+//   calculator: "RegexPreprocessorCalculator"
+//   input_stream: "TEXT:text"
+//   input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
+//   output_stream: "TENSORS:tensors"
+//   options {
+//     [mediapipe.RegexPreprocessorCalculatorOptions.ext] {
+//       max_seq_len: 256
+//     }
+//   }
+// }
+class RegexPreprocessorCalculator : public Node {
+ public:
+  static constexpr Input<std::string> kTextIn{"TEXT"};
+  static constexpr SideInput<ModelMetadataExtractor> kMetadataExtractorSideIn{
+      "METADATA_EXTRACTOR"};
+  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
+
+  MEDIAPIPE_NODE_CONTRACT(kTextIn, kMetadataExtractorSideIn, kTensorsOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  std::unique_ptr<tasks::text::tokenizers::RegexTokenizer> tokenizer_;
+  // The max sequence length accepted by the text model.
+  int max_seq_len_ = 0;
+};
+
+absl::Status RegexPreprocessorCalculator::UpdateContract(
+    CalculatorContract* cc) {
+  const auto& options =
+      cc->Options<mediapipe::RegexPreprocessorCalculatorOptions>();
+  RET_CHECK(options.has_max_seq_len()) << "max_seq_len is required";
+  RET_CHECK_GT(options.max_seq_len(), 0) << "max_seq_len must be positive";
+  return absl::OkStatus();
+}
+
+absl::Status RegexPreprocessorCalculator::Open(CalculatorContext* cc) {
+  const ModelMetadataExtractor* metadata_extractor =
+      &kMetadataExtractorSideIn(cc).Get();
+  const tflite::TensorMetadata* tensor_metadata =
+      metadata_extractor->GetInputTensorMetadata(0);
+  if (tensor_metadata == nullptr) {
+    return absl::InvalidArgumentError("No tensor metadata found");
+  }
+
+  ASSIGN_OR_RETURN(
+      const auto* tokenizer_metadata,
+      metadata_extractor->FindFirstProcessUnit(
+          *tensor_metadata, tflite::ProcessUnitOptions_RegexTokenizerOptions));
+  if (tokenizer_metadata == nullptr) {
+    return absl::InvalidArgumentError("No tokenizer metadata found");
+  }
+  const tflite::RegexTokenizerOptions* regex_tokenizer_options =
+      tokenizer_metadata->options_as<tflite::RegexTokenizerOptions>();
+  ASSIGN_OR_RETURN(tokenizer_,
+                   tasks::text::tokenizers::CreateRegexTokenizerFromOptions(
+                       regex_tokenizer_options, metadata_extractor));
+
+  const auto& options =
+      cc->Options<mediapipe::RegexPreprocessorCalculatorOptions>();
+  max_seq_len_ = options.max_seq_len();
+  return absl::OkStatus();
+}
+
+absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
+  tasks::text::tokenizers::TokenizerResult tokenizer_result =
+      tokenizer_->Tokenize(kTextIn(cc).Get());
+
+  int unknown_token_id = 0;
+  tokenizer_->GetUnknownToken(&unknown_token_id);
+  int pad_token_id = 0;
+  tokenizer_->GetPadToken(&pad_token_id);
+
+  std::vector<int> input_tokens(max_seq_len_, pad_token_id);
+  int start_token_id = 0;
+  int input_token_index = 0;
+  if (tokenizer_->GetStartToken(&start_token_id)) {
+    input_tokens[0] = start_token_id;
+    input_token_index = 1;
+  }
+
+  for (int i = 0; (i < tokenizer_result.subwords.size()) &&
+                  (input_token_index < max_seq_len_);
+       ++i, ++input_token_index) {
+    const std::string& token = tokenizer_result.subwords[i];
+    int token_id = 0;
+    if (tokenizer_->LookupId(token, &token_id)) {
+      input_tokens[input_token_index] = token_id;
+    } else {
+      input_tokens[input_token_index] = unknown_token_id;
+    }
+  }
+
+  //                              |<-------sentence_length-------->|
+  // input_tensor                 <START>, t1, t2... <PAD>, <PAD>...
+  // <START> is optional, t1, t2... will be replaced by <UNKNOWN> if it's
+  // not found in the tokenizer vocab.
+  std::vector<Tensor> result;
+  result.push_back(
+      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
+  std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
+              input_tokens.data(), input_tokens.size() * sizeof(int32_t));
+  kTensorsOut(cc).Send(std::move(result));
+  return absl::OkStatus();
+}
+
+MEDIAPIPE_REGISTER_NODE(RegexPreprocessorCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/regex_preprocessor_calculator.proto

@@ -0,0 +1,29 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message RegexPreprocessorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional RegexPreprocessorCalculatorOptions ext = 463716697;
+  }
+
+  // The maximum input sequence length for the calculator's text model.
+  optional int32 max_seq_len = 1;
+}

mediapipe/calculators/tensor/regex_preprocessor_calculator_test.cc

@@ -0,0 +1,130 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/sink.h"
+#include "mediapipe/tasks/cc/core/utils.h"
+#include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
+using ::testing::ElementsAreArray;
+
+constexpr int kMaxSeqLen = 256;
+constexpr char kTestModelPath[] =
+    "mediapipe/tasks/testdata/text/"
+    "test_model_text_classifier_with_regex_tokenizer.tflite";
+
+absl::StatusOr<std::vector<int>> RunRegexPreprocessorCalculator(
+    absl::string_view text) {
+  auto graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
+          R"pb(
+            input_stream: "text"
+            output_stream: "tensors"
+            node {
+              calculator: "RegexPreprocessorCalculator"
+              input_stream: "TEXT:text"
+              input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
+              output_stream: "TENSORS:tensors"
+              options {
+                [mediapipe.RegexPreprocessorCalculatorOptions.ext] {
+                  max_seq_len: $0
+                }
+              }
+            }
+          )pb",
+          kMaxSeqLen));
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("tensors", &graph_config, &output_packets);
+
+  std::string model_buffer = tasks::core::LoadBinaryContent(kTestModelPath);
+  ASSIGN_OR_RETURN(std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
+                   ModelMetadataExtractor::CreateFromModelBuffer(
+                       model_buffer.data(), model_buffer.size()));
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_RETURN_IF_ERROR(graph.Initialize(
+      graph_config,
+      {{"metadata_extractor",
+        MakePacket<ModelMetadataExtractor>(std::move(*metadata_extractor))}}));
+  MP_RETURN_IF_ERROR(graph.StartRun({}));
+  MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
+      "text", MakePacket<std::string>(text).At(Timestamp(0))));
+  MP_RETURN_IF_ERROR(graph.WaitUntilIdle());
+
+  if (output_packets.size() != 1) {
+    return absl::InvalidArgumentError(absl::Substitute(
+        "output_packets has size $0, expected 1", output_packets.size()));
+  }
+  const std::vector<Tensor>& tensor_vec =
+      output_packets[0].Get<std::vector<Tensor>>();
+  if (tensor_vec.size() != 1) {
+    return absl::InvalidArgumentError(absl::Substitute(
+        "tensor_vec has size $0, expected $1", tensor_vec.size(), 1));
+  }
+  if (tensor_vec[0].element_type() != Tensor::ElementType::kInt32) {
+    return absl::InvalidArgumentError("Expected tensor element type kInt32");
+  }
+  auto* buffer = tensor_vec[0].GetCpuReadView().buffer<int>();
+  std::vector<int> result(buffer, buffer + kMaxSeqLen);
+  MP_RETURN_IF_ERROR(graph.CloseAllPacketSources());
+  MP_RETURN_IF_ERROR(graph.WaitUntilDone());
+  return result;
+}
+
+TEST(RegexPreprocessorCalculatorTest, TextClassifierModel) {
+  MP_ASSERT_OK_AND_ASSIGN(
+      std::vector<int> processed_tensor_values,
+      RunRegexPreprocessorCalculator("This is the best movie I’ve seen in "
+                                     "recent years. Strongly recommend it!"));
+  static const int expected_result[kMaxSeqLen] = {
+      1, 2, 9, 4, 118, 20, 2, 2, 110, 11, 1136, 153, 2, 386, 12};
+  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
+}
+
+TEST(RegexPreprocessorCalculatorTest, LongInput) {
+  std::stringstream long_input;
+  long_input << "This is the best";
+  for (int i = 0; i < kMaxSeqLen; ++i) {
+    long_input << " best";
+  }
+  long_input << "movie I’ve seen in recent years. Strongly recommend it!";
+  MP_ASSERT_OK_AND_ASSIGN(std::vector<int> processed_tensor_values,
+                          RunRegexPreprocessorCalculator(long_input.str()));
+  std::vector<int> expected_result = {1, 2, 9, 4, 118};
+  // "best" id
+  expected_result.resize(kMaxSeqLen, 118);
+  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensor_converter_calculator.cc

@@ -31,6 +31,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 
+#include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 #include "mediapipe/gpu/gl_calculator_helper.h"
@@ -296,7 +297,6 @@ absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) {
   output_tensors->emplace_back(Tensor::ElementType::kFloat32,
                                Tensor::Shape{1, height, width, channels});
 #if MEDIAPIPE_METAL_ENABLED
-  id<MTLDevice> device = gpu_helper_.mtlDevice;
   id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
   command_buffer.label = @"TensorConverterCalculatorConvert";
   id<MTLComputeCommandEncoder> compute_encoder =
@@ -305,7 +305,7 @@ absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) {
   id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
   [compute_encoder setTexture:src_texture atIndex:0];
   auto output_view =
-      output_tensors->at(0).GetMtlBufferWriteView(command_buffer);
+      MtlBufferView::GetWriteView(output_tensors->at(0), command_buffer);
   [compute_encoder setBuffer:output_view.buffer() offset:0 atIndex:1];
   MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
   MTLSize threadgroups =

mediapipe/calculators/tensor/tensors_dequantization_calculator.cc

@@ -87,6 +87,9 @@ absl::Status TensorsDequantizationCalculator::Process(CalculatorContext* cc) {
       case Tensor::ElementType::kInt8:
         Dequantize<int8>(input_tensor, &output_tensors->back());
         break;
+      case Tensor::ElementType::kBool:
+        Dequantize<bool>(input_tensor, &output_tensors->back());
+        break;
       default:
         return absl::InvalidArgumentError(absl::StrCat(
             "Unsupported input tensor type: ", input_tensor.element_type()));

mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc

@@ -124,5 +124,15 @@ TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithInt8Tensors) {
   ValidateResult(GetOutput(), {-1.007874, 0, 1});
 }
 
+TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithBoolTensors) {
+  std::vector<bool> tensor = {true, false, true};
+  PushTensor(Tensor::ElementType::kBool, tensor,
+             Tensor::QuantizationParameters{1.0f, 0});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  ValidateResult(GetOutput(), {1, 0, 1});
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_audio_calculator.cc

@@ -0,0 +1,197 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <new>
+#include <utility>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "absl/status/status.h"
+#include "audio/dsp/window_functions.h"
+#include "mediapipe/calculators/tensor/tensors_to_audio_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "pffft.h"
+
+namespace mediapipe {
+namespace api2 {
+namespace {
+
+std::vector<float> HannWindow(int window_size, bool sqrt_hann) {
+  std::vector<float> hann_window(window_size);
+  audio_dsp::HannWindow().GetPeriodicSamples(window_size, &hann_window);
+  if (sqrt_hann) {
+    absl::c_transform(hann_window, hann_window.begin(),
+                      [](double x) { return std::sqrt(x); });
+  }
+  return hann_window;
+}
+
+// Note that the InvHannWindow function may only work for 50% overlapping case.
+std::vector<float> InvHannWindow(int window_size, bool sqrt_hann) {
+  std::vector<float> window = HannWindow(window_size, sqrt_hann);
+  std::vector<float> inv_window(window.size());
+  if (sqrt_hann) {
+    absl::c_copy(window, inv_window.begin());
+  } else {
+    const int kHalfWindowSize = window.size() / 2;
+    absl::c_transform(window, inv_window.begin(),
+                      [](double x) { return x * x; });
+    for (int i = 0; i < kHalfWindowSize; ++i) {
+      double sum = inv_window[i] + inv_window[kHalfWindowSize + i];
+      inv_window[i] = window[i] / sum;
+      inv_window[kHalfWindowSize + i] = window[kHalfWindowSize + i] / sum;
+    }
+  }
+  return inv_window;
+}
+
+// PFFFT only supports transforms for inputs of length N of the form
+// N = (2^a)*(3^b)*(5^c) where b >=0 and c >= 0 and a >= 5 for the real FFT.
+bool IsValidFftSize(int size) {
+  if (size <= 0) {
+    return false;
+  }
+  constexpr int kFactors[] = {2, 3, 5};
+  int factorization[] = {0, 0, 0};
+  int n = static_cast<int>(size);
+  for (int i = 0; i < 3; ++i) {
+    while (n % kFactors[i] == 0) {
+      n = n / kFactors[i];
+      ++factorization[i];
+    }
+  }
+  return factorization[0] >= 5 && n == 1;
+}
+
+}  // namespace
+
+// Converts 2D MediaPipe float Tensors to audio buffers.
+// The calculator will perform ifft on the complex DFT and apply the window
+// function (Inverse Hann) afterwards. The input 2D MediaPipe Tensor must
+// have the DFT real parts in its first row and the DFT imagery parts in its
+// second row. A valid "fft_size" must be set in the CalculatorOptions.
+//
+// Inputs:
+//   TENSORS - std::vector<Tensor>
+//     Vector containing a single Tensor that represents the audio's complex DFT
+//     results.
+//   DC_AND_NYQUIST - std::pair<float, float>
+//     A pair of dc component and nyquist component.
+//
+// Outputs:
+//   AUDIO - mediapipe::Matrix
+//     The audio data represented as mediapipe::Matrix.
+//
+// Example:
+// node {
+//   calculator: "TensorsToAudioCalculator"
+//   input_stream: "TENSORS:tensors"
+//   input_stream: "DC_AND_NYQUIST:dc_and_nyquist"
+//   output_stream: "AUDIO:audio"
+//   options {
+//     [mediapipe.AudioToTensorCalculatorOptions.ext] {
+//       fft_size: 256
+//     }
+//   }
+// }
+class TensorsToAudioCalculator : public Node {
+ public:
+  static constexpr Input<std::vector<Tensor>> kTensorsIn{"TENSORS"};
+  static constexpr Input<std::pair<float, float>> kDcAndNyquistIn{
+      "DC_AND_NYQUIST"};
+  static constexpr Output<Matrix> kAudioOut{"AUDIO"};
+  MEDIAPIPE_NODE_CONTRACT(kTensorsIn, kDcAndNyquistIn, kAudioOut);
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+  absl::Status Close(CalculatorContext* cc) override;
+
+ private:
+  // The internal state of the FFT library.
+  PFFFT_Setup* fft_state_ = nullptr;
+  int fft_size_ = 0;
+  float inverse_fft_size_ = 0;
+  std::vector<float, Eigen::aligned_allocator<float>> input_dft_;
+  std::vector<float> inv_fft_window_;
+  std::vector<float, Eigen::aligned_allocator<float>> fft_input_buffer_;
+  // pffft requires memory to work with to avoid using the stack.
+  std::vector<float, Eigen::aligned_allocator<float>> fft_workplace_;
+  std::vector<float, Eigen::aligned_allocator<float>> fft_output_;
+};
+
+absl::Status TensorsToAudioCalculator::Open(CalculatorContext* cc) {
+  const auto& options =
+      cc->Options<mediapipe::TensorsToAudioCalculatorOptions>();
+  RET_CHECK(options.has_fft_size()) << "FFT size must be specified.";
+  RET_CHECK(IsValidFftSize(options.fft_size()))
+      << "FFT size must be of the form fft_size = (2^a)*(3^b)*(5^c) where b "
+         ">=0 and c >= 0 and a >= 5, the requested fft size is "
+      << options.fft_size();
+  fft_size_ = options.fft_size();
+  inverse_fft_size_ = 1.0f / fft_size_;
+  fft_state_ = pffft_new_setup(fft_size_, PFFFT_REAL);
+  input_dft_.resize(fft_size_);
+  inv_fft_window_ = InvHannWindow(fft_size_, /* sqrt_hann = */ false);
+  fft_input_buffer_.resize(fft_size_);
+  fft_workplace_.resize(fft_size_);
+  fft_output_.resize(fft_size_);
+  return absl::OkStatus();
+}
+
+absl::Status TensorsToAudioCalculator::Process(CalculatorContext* cc) {
+  if (kTensorsIn(cc).IsEmpty() || kDcAndNyquistIn(cc).IsEmpty()) {
+    return absl::OkStatus();
+  }
+  const auto& input_tensors = *kTensorsIn(cc);
+  RET_CHECK_EQ(input_tensors.size(), 1);
+  RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
+  auto view = input_tensors[0].GetCpuReadView();
+  // DC's real part.
+  input_dft_[0] = kDcAndNyquistIn(cc)->first;
+  // Nyquist's real part is the penultimate element of the tensor buffer.
+  // pffft ignores the Nyquist's imagery part. No need to fetch the last value
+  // from the tensor buffer.
+  input_dft_[1] = *(view.buffer<float>() + (fft_size_ - 2));
+  std::memcpy(input_dft_.data() + 2, view.buffer<float>(),
+              (fft_size_ - 2) * sizeof(float));
+  pffft_transform_ordered(fft_state_, input_dft_.data(), fft_output_.data(),
+                          fft_workplace_.data(), PFFFT_BACKWARD);
+  // Applies the inverse window function.
+  std::transform(
+      fft_output_.begin(), fft_output_.end(), inv_fft_window_.begin(),
+      fft_output_.begin(),
+      [this](float a, float b) { return a * b * inverse_fft_size_; });
+  Matrix matrix = Eigen::Map<Matrix>(fft_output_.data(), 1, fft_output_.size());
+  kAudioOut(cc).Send(std::move(matrix));
+  return absl::OkStatus();
+}
+
+absl::Status TensorsToAudioCalculator::Close(CalculatorContext* cc) {
+  if (fft_state_) {
+    pffft_destroy_setup(fft_state_);
+  }
+  return absl::OkStatus();
+}
+
+MEDIAPIPE_REGISTER_NODE(TensorsToAudioCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_audio_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message TensorsToAudioCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional TensorsToAudioCalculatorOptions ext = 484297136;
+  }
+
+  // Size of the fft in number of bins. If set, the calculator will do ifft
+  // on the input tensor.
+  optional int64 fft_size = 1;
+}

mediapipe/calculators/tensor/tensors_to_audio_calculator_test.cc

@@ -0,0 +1,149 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <new>
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensors_to_audio_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+class TensorsToAudioCalculatorFftTest : public ::testing::Test {
+ protected:
+  // Creates an audio matrix containing a single sample of 1.0 at a specified
+  // offset.
+  Matrix CreateImpulseSignalData(int64 num_samples, int impulse_offset_idx) {
+    Matrix impulse = Matrix::Zero(1, num_samples);
+    impulse(0, impulse_offset_idx) = 1.0;
+    return impulse;
+  }
+
+  void ConfigGraph(int num_samples, double sample_rate, int fft_size) {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        absl::Substitute(R"(
+        input_stream: "audio_in"
+        input_stream: "sample_rate"
+        output_stream: "audio_out"
+        node {
+          calculator: "AudioToTensorCalculator"
+          input_stream: "AUDIO:audio_in"
+          input_stream: "SAMPLE_RATE:sample_rate"
+          output_stream: "TENSORS:tensors"
+          output_stream: "DC_AND_NYQUIST:dc_and_nyquist"
+          options {
+            [mediapipe.AudioToTensorCalculatorOptions.ext] {
+              num_channels: 1
+              num_samples: $0
+              num_overlapping_samples: 0
+              target_sample_rate: $1
+              fft_size: $2
+            }
+          }
+        }
+        node {
+          calculator: "TensorsToAudioCalculator"
+          input_stream: "TENSORS:tensors"
+          input_stream: "DC_AND_NYQUIST:dc_and_nyquist"
+          output_stream: "AUDIO:audio_out"
+          options {
+            [mediapipe.TensorsToAudioCalculatorOptions.ext] {
+              fft_size: $2
+            }
+          }
+        }
+        )",
+                         /*$0=*/num_samples,
+                         /*$1=*/sample_rate,
+                         /*$2=*/fft_size));
+    tool::AddVectorSink("audio_out", &graph_config_, &audio_out_packets_);
+  }
+
+  void RunGraph(const Matrix& input_data, double sample_rate) {
+    MP_ASSERT_OK(graph_.Initialize(graph_config_));
+    MP_ASSERT_OK(graph_.StartRun({}));
+    MP_ASSERT_OK(graph_.AddPacketToInputStream(
+        "sample_rate", MakePacket<double>(sample_rate).At(Timestamp(0))));
+    MP_ASSERT_OK(graph_.AddPacketToInputStream(
+        "audio_in", MakePacket<Matrix>(input_data).At(Timestamp(0))));
+    MP_ASSERT_OK(graph_.CloseAllInputStreams());
+    MP_ASSERT_OK(graph_.WaitUntilDone());
+  }
+
+  std::vector<Packet> audio_out_packets_;
+  CalculatorGraphConfig graph_config_;
+  CalculatorGraph graph_;
+};
+
+TEST_F(TensorsToAudioCalculatorFftTest, TestInvalidFftSize) {
+  ConfigGraph(320, 16000, 103);
+  MP_ASSERT_OK(graph_.Initialize(graph_config_));
+  MP_ASSERT_OK(graph_.StartRun({}));
+  auto status = graph_.WaitUntilIdle();
+  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
+  EXPECT_THAT(status.message(),
+              ::testing::HasSubstr("FFT size must be of the form"));
+}
+
+TEST_F(TensorsToAudioCalculatorFftTest, TestImpulseSignalAtTheCenter) {
+  constexpr int sample_size = 320;
+  constexpr double sample_rate = 16000;
+  ConfigGraph(sample_size, sample_rate, 320);
+
+  Matrix impulse_data = CreateImpulseSignalData(sample_size, sample_size / 2);
+  RunGraph(impulse_data, sample_rate);
+  ASSERT_EQ(1, audio_out_packets_.size());
+  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
+  // The impulse signal at the center is not affected by the window function.
+  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), impulse_data);
+}
+
+TEST_F(TensorsToAudioCalculatorFftTest, TestWindowedImpulseSignal) {
+  constexpr int sample_size = 320;
+  constexpr double sample_rate = 16000;
+  ConfigGraph(sample_size, sample_rate, 320);
+  Matrix impulse_data = CreateImpulseSignalData(sample_size, sample_size / 4);
+  RunGraph(impulse_data, sample_rate);
+  ASSERT_EQ(1, audio_out_packets_.size());
+  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
+  // As the impulse signal sits at the 1/4 of the hann window, the inverse
+  // window function reduces it by half.
+  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), impulse_data / 2);
+}
+
+TEST_F(TensorsToAudioCalculatorFftTest, TestImpulseSignalAtBeginning) {
+  constexpr int sample_size = 320;
+  constexpr double sample_rate = 16000;
+  ConfigGraph(sample_size, sample_rate, 320);
+  Matrix impulse_data = CreateImpulseSignalData(sample_size, 0);
+  RunGraph(impulse_data, sample_rate);
+  ASSERT_EQ(1, audio_out_packets_.size());
+  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
+  // As the impulse signal sits at the beginning of the hann window, the inverse
+  // window function completely removes it.
+  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), Matrix::Zero(1, sample_size));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -41,6 +41,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 
+#include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
@@ -532,15 +533,15 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
                                          detection_classes.data(),
                                          output_detections));
 #elif MEDIAPIPE_METAL_ENABLED
-  id<MTLDevice> device = gpu_helper_.mtlDevice;
   if (!anchors_init_) {
     if (input_tensors.size() == kNumInputTensorsWithAnchors) {
       RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors);
       auto command_buffer = [gpu_helper_ commandBuffer];
-      auto src_buffer = input_tensors[tensor_mapping_.anchors_tensor_index()]
-                            .GetMtlBufferReadView(command_buffer);
+      auto src_buffer = MtlBufferView::GetReadView(
+          input_tensors[tensor_mapping_.anchors_tensor_index()],
+          command_buffer);
       auto dest_buffer =
-          raw_anchors_buffer_->GetMtlBufferWriteView(command_buffer);
+          MtlBufferView::GetWriteView(*raw_anchors_buffer_, command_buffer);
       id<MTLBlitCommandEncoder> blit_command =
           [command_buffer blitCommandEncoder];
       [blit_command copyFromBuffer:src_buffer.buffer()
@@ -572,15 +573,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
   [command_encoder setComputePipelineState:decode_program_];
   {
     auto scored_boxes_view =
-        scored_boxes_buffer_->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*scored_boxes_buffer_, command_buffer);
     auto decoded_boxes_view =
-        decoded_boxes_buffer_->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*decoded_boxes_buffer_, command_buffer);
     [command_encoder setBuffer:decoded_boxes_view.buffer() offset:0 atIndex:0];
-    auto input0_view = input_tensors[tensor_mapping_.detections_tensor_index()]
-                           .GetMtlBufferReadView(command_buffer);
+    auto input0_view = MtlBufferView::GetReadView(
+        input_tensors[tensor_mapping_.detections_tensor_index()],
+        command_buffer);
     [command_encoder setBuffer:input0_view.buffer() offset:0 atIndex:1];
     auto raw_anchors_view =
-        raw_anchors_buffer_->GetMtlBufferReadView(command_buffer);
+        MtlBufferView::GetReadView(*raw_anchors_buffer_, command_buffer);
     [command_encoder setBuffer:raw_anchors_view.buffer() offset:0 atIndex:2];
     MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
     MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
@@ -589,8 +591,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
 
     [command_encoder setComputePipelineState:score_program_];
     [command_encoder setBuffer:scored_boxes_view.buffer() offset:0 atIndex:0];
-    auto input1_view = input_tensors[tensor_mapping_.scores_tensor_index()]
-                           .GetMtlBufferReadView(command_buffer);
+    auto input1_view = MtlBufferView::GetReadView(
+        input_tensors[tensor_mapping_.scores_tensor_index()], command_buffer);
     [command_encoder setBuffer:input1_view.buffer() offset:0 atIndex:1];
     MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
     MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);

mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc

@@ -53,6 +53,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 
+#include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
@@ -485,7 +486,8 @@ absl::Status TensorsToSegmentationCalculator::ProcessGpu(
         [command_buffer computeCommandEncoder];
     [command_encoder setComputePipelineState:mask_program_];
 
-    auto read_view = input_tensors[0].GetMtlBufferReadView(command_buffer);
+    auto read_view =
+        MtlBufferView::GetReadView(input_tensors[0], command_buffer);
     [command_encoder setBuffer:read_view.buffer() offset:0 atIndex:0];
 
     mediapipe::GpuBuffer small_mask_buffer = [metal_helper_