Internal change

PiperOrigin-RevId: 504341886
Internal change
2023-01-24 12:18:15 -08:00 · 2023-01-24 12:14:02 -08:00 · 2023-01-24 10:38:25 -08:00 · 2023-01-23 16:43:23 -08:00 · 2023-01-23 14:15:21 -08:00 · 2023-01-23 13:38:41 -08:00
1206 changed files with 131285 additions and 8157 deletions
@@ -0,0 +1,25 @@
 ---
 name: "Tasks Issue"
 about: Use this template for assistance with using MediaPipe Tasks (developers.google.com/mediapipe/solutions) to deploy on-device ML solutions (e.g. gesture recognition etc.) on supported platforms.
 labels: type:support
 ---
 <em>Please make sure that this is a [Tasks](https://developers.google.com/mediapipe/solutions) issue.<em>
 **System information** (Please provide as much relevant information as possible)
 - Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
 - OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
 - MediaPipe Tasks SDK version:
 - Task name (e.g. Object detection, Gesture recognition etc.):
 - Programming Language and version ( e.g. C++, Python, Java):
 **Describe the expected behavior:**
 **Standalone code you may have used to try to get what you need :**
 If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
 **Other info / Complete Logs :**
 Include any logs or source code that would be helpful to
 diagnose the problem. If including tracebacks, please include the full
 traceback. Large logs and files should be attached:
@@ -0,0 +1,25 @@
 ---
 name: "Model Maker Issue"
 about: Use this template for assistance with using MediaPipe Model Maker (developers.google.com/mediapipe/solutions) to create custom on-device ML solutions.
 labels: type:support
 ---
 <em>Please make sure that this is a [Model Maker](https://developers.google.com/mediapipe/solutions) issue.<em>
 **System information** (Please provide as much relevant information as possible)
 - Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
 - OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
 - Python version (e.g. 3.8):
 - [MediaPipe Model Maker version](https://pypi.org/project/mediapipe-model-maker/):
 - Task name (e.g. Image classification, Gesture recognition etc.):
 **Describe the expected behavior:**
 **Standalone code you may have used to try to get what you need :**
 If there is a problem, provide a reproducible test case that is the bare minimum necessary to generate the problem. If possible, please share a link to Colab, GitHub repo link or anything that we can use to reproduce the problem:
 **Other info / Complete Logs :**
 Include any logs or source code that would be helpful to
 diagnose the problem. If including tracebacks, please include the full
 traceback. Large logs and files should be attached:
@@ -1,6 +1,6 @@
 ---
-name: "Solution Issue"
+name: "Solution (legacy) Issue"
-about: Use this template for assistance with a specific mediapipe solution, such as "Pose" or "Iris", including inference model usage/training, solution-specific calculators, etc.
+about: Use this template for assistance with a specific Mediapipe solution (google.github.io/mediapipe/solutions) such as "Pose", including inference model usage/training, solution-specific calculators etc.
 labels: type:support
 ---
@@ -0,0 +1,19 @@
 ---
 name: "Studio Issue"
 about: Use this template for assistance with the MediaPipe Studio application.
 labels: type:support
 ---
 <em>Please make sure that this is a MediaPipe Studio issue.<em>
 **System information** (Please provide as much relevant information as possible)
 - OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
 - Browser and Version
 - Any microphone or camera hardware
 - URL that shows the problem
 **Describe the expected behavior:**
 **Other info / Complete Logs :**
 Include any js console logs that would be helpful to diagnose the problem.
 Large logs and files should be attached:
@@ -15,4 +15,5 @@
 # A list of assignees
 assignees:
-   - sureshdagooglecom
+   - kuaashish
   - ayushgdev
@@ -2,5 +2,6 @@ bazel-*
 mediapipe/MediaPipe.xcodeproj
 mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user
 mediapipe/provisioning_profile.mobileprovision
 node_modules/
 .configure.bazelrc
 .user.bazelrc
@@ -1,4 +1,4 @@
-# Copyright 2019 The MediaPipe Authors.
+# Copyright 2022 The MediaPipe Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,4 +14,9 @@
 licenses(["notice"])
-exports_files(["LICENSE"])
+exports_files([
    "LICENSE",
    "tsconfig.json",
    "package.json",
    "yarn.lock",
 ])
@@ -30,6 +30,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
        git \
        wget \
        unzip \
        nodejs \
        npm \
        python3-dev \
        python3-opencv \
        python3-pip \
@@ -53,7 +55,7 @@ RUN pip3 install wheel
 RUN pip3 install future
 RUN pip3 install absl-py numpy opencv-contrib-python protobuf==3.20.1
 RUN pip3 install six==1.14.0
-RUN pip3 install tensorflow==2.2.0
+RUN pip3 install tensorflow
 RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
@@ -26,7 +26,7 @@ versions.check(minimum_bazel_version = "3.7.2")
 http_archive(
    name = "com_google_absl",
    urls = [
-        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/refs/tags/20220623.1.tar.gz",
    ],
    # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
    patches = [
@@ -35,8 +35,8 @@ http_archive(
    patch_args = [
        "-p1",
    ],
-    strip_prefix = "abseil-cpp-20210324.2",
+    strip_prefix = "abseil-cpp-20220623.1",
-    sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f"
+    sha256 = "91ac87d30cc6d79f9ab974c51874a704de9c2647c40f6932597329a282217ba8"
 )
 http_archive(
@@ -155,6 +155,14 @@ http_archive(
    name = "com_google_audio_tools",
    strip_prefix = "multichannel-audio-tools-master",
    urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
    repo_mapping = {"@com_github_glog_glog" : "@com_github_glog_glog_no_gflags"},
 )
 http_archive(
    name = "pffft",
    strip_prefix = "jpommier-pffft-7c3b5a7dc510",
    urls = ["https://bitbucket.org/jpommier/pffft/get/7c3b5a7dc510.zip"],
    build_file = "@//third_party:pffft.BUILD",
 )
 # sentencepiece
@@ -165,7 +173,11 @@ http_archive(
    urls = [
        "https://github.com/google/sentencepiece/archive/1.0.0.zip",
    ],
-    repo_mapping = {"@com_google_glog" : "@com_github_glog_glog"},
+    patches = [
        "//third_party:com_google_sentencepiece_no_gflag_no_gtest.diff",
    ],
    patch_args = ["-p1"],
    repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
 )
 http_archive(
@@ -200,14 +212,14 @@ http_archive(
    sha256 = "75922da3a1bdb417d820398eb03d4e9bd067c4905a4246d35a44c01d62154d91",
 )
-# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
+# 2022-10-20
 http_archive(
    name = "pybind11",
    urls = [
-        "https://github.com/pybind/pybind11/archive/70a58c577eaf067748c2ec31bfd0b0a614cffba6.zip",
+        "https://github.com/pybind/pybind11/archive/v2.10.1.zip",
    ],
-    sha256 = "b971842fab1b5b8f3815a2302331782b7d137fef0e06502422bc4bc360f4956c",
+    sha256 = "fcf94065efcfd0a7a828bacf118fa11c43f6390d0c805e3e6342ac119f2e9976",
-    strip_prefix = "pybind11-70a58c577eaf067748c2ec31bfd0b0a614cffba6",
+    strip_prefix = "pybind11-2.10.1",
    build_file = "@pybind11_bazel//:pybind11.BUILD",
 )
@@ -308,12 +320,30 @@ http_archive(
    ],
 )
-# iOS basic build deps.
+# Load Zlib before initializing TensorFlow and the iOS build rules to guarantee
 # that the target @zlib//:mini_zlib is available
 http_archive(
    name = "zlib",
    build_file = "//third_party:zlib.BUILD",
    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
    strip_prefix = "zlib-1.2.11",
    urls = [
        "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
        "http://zlib.net/fossils/zlib-1.2.11.tar.gz",  # 2017-01-15
    ],
    patches = [
        "@//third_party:zlib.diff",
    ],
    patch_args = [
        "-p1",
    ],
 )
 # iOS basic build deps.
 http_archive(
    name = "build_bazel_rules_apple",
-    sha256 = "77e8bf6fda706f420a55874ae6ee4df0c9d95da6c7838228b26910fc82eea5a2",
+    sha256 = "f94e6dddf74739ef5cb30f000e13a2a613f6ebfa5e63588305a71fce8a8a9911",
-    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.32.0/rules_apple.0.32.0.tar.gz",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/1.1.3/rules_apple.1.1.3.tar.gz",
    patches = [
        # Bypass checking ios unit test runner when building MP ios applications.
        "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
@@ -327,29 +357,24 @@ load(
    "@build_bazel_rules_apple//apple:repositories.bzl",
    "apple_rules_dependencies",
 )
 apple_rules_dependencies()
 load(
    "@build_bazel_rules_swift//swift:repositories.bzl",
    "swift_rules_dependencies",
 )
 swift_rules_dependencies()
-http_archive(
+load(
-    name = "build_bazel_apple_support",
+    "@build_bazel_rules_swift//swift:extras.bzl",
-    sha256 = "741366f79d900c11e11d8efd6cc6c66a31bfb2451178b58e0b5edc6f1db17b35",
+    "swift_rules_extra_dependencies",
    urls = [
        "https://github.com/bazelbuild/apple_support/releases/download/0.10.0/apple_support.0.10.0.tar.gz"
    ],
 )
 swift_rules_extra_dependencies()
 load(
    "@build_bazel_apple_support//lib:repositories.bzl",
    "apple_support_dependencies",
 )
 apple_support_dependencies()
 # More iOS deps.
@@ -430,25 +455,6 @@ http_archive(
    ],
 )
 # Load Zlib before initializing TensorFlow to guarantee that the target
 # @zlib//:mini_zlib is available
 http_archive(
    name = "zlib",
    build_file = "//third_party:zlib.BUILD",
    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
    strip_prefix = "zlib-1.2.11",
    urls = [
        "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
        "http://zlib.net/fossils/zlib-1.2.11.tar.gz",  # 2017-01-15
    ],
    patches = [
        "@//third_party:zlib.diff",
    ],
    patch_args = [
        "-p1",
    ],
 )
 # TensorFlow repo should always go after the other external dependencies.
 # TF on 2022-08-10.
 _TENSORFLOW_GIT_COMMIT = "af1d5bc4fbb66d9e6cc1cf89503014a99233583b"
@@ -490,5 +496,51 @@ libedgetpu_dependencies()
 load("@coral_crosstool//:configure.bzl", "cc_crosstool")
 cc_crosstool(name = "crosstool")
 # Node dependencies
 http_archive(
    name = "build_bazel_rules_nodejs",
    sha256 = "5aae76dced38f784b58d9776e4ab12278bc156a9ed2b1d9fcd3e39921dc88fda",
    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/5.7.1/rules_nodejs-5.7.1.tar.gz"],
 )
 load("@build_bazel_rules_nodejs//:repositories.bzl", "build_bazel_rules_nodejs_dependencies")
 build_bazel_rules_nodejs_dependencies()
 # fetches nodejs, npm, and yarn
 load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install")
 node_repositories()
 yarn_install(
    name = "npm",
    package_json = "//:package.json",
    yarn_lock = "//:yarn.lock",
 )
 # Protobuf for Node dependencies
 http_archive(
    name = "rules_proto_grpc",
    sha256 = "bbe4db93499f5c9414926e46f9e35016999a4e9f6e3522482d3760dc61011070",
    strip_prefix = "rules_proto_grpc-4.2.0",
    urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/4.2.0.tar.gz"],
 )
 http_archive(
    name = "com_google_protobuf_javascript",
    sha256 = "35bca1729532b0a77280bf28ab5937438e3dcccd6b31a282d9ae84c896b6f6e3",
    strip_prefix = "protobuf-javascript-3.21.2",
    urls = ["https://github.com/protocolbuffers/protobuf-javascript/archive/refs/tags/v3.21.2.tar.gz"],
 )
 load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains", "rules_proto_grpc_repos")
 rules_proto_grpc_toolchains()
 rules_proto_grpc_repos()
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 rules_proto_dependencies()
 rules_proto_toolchains()
 load("//third_party:external_files.bzl", "external_files")
 external_files()
 load("//third_party:wasm_files.bzl", "wasm_files")
 wasm_files()
@@ -0,0 +1,33 @@
 # Placeholder for internal Python strict binary compatibility macro.
 py_binary(
    name = "build_py_api_docs",
    srcs = ["build_py_api_docs.py"],
    deps = [
        "//mediapipe",
        "//third_party/py/absl:app",
        "//third_party/py/absl/flags",
        "//third_party/py/tensorflow_docs",
        "//third_party/py/tensorflow_docs/api_generator:generate_lib",
        "//third_party/py/tensorflow_docs/api_generator:public_api",
    ],
 )
 py_binary(
    name = "build_java_api_docs",
    srcs = ["build_java_api_docs.py"],
    data = [
        "//third_party/android/sdk:api/26.txt",
        "//third_party/java/doclava/current:doclava.jar",
        "//third_party/java/jsilver:jsilver_jar",
    ],
    env = {
        "DOCLAVA_JAR": "$(location //third_party/java/doclava/current:doclava.jar)",
        "JSILVER_JAR": "$(location //third_party/java/jsilver:jsilver_jar)",
    },
    deps = [
        "//third_party/py/absl:app",
        "//third_party/py/absl/flags",
        "//third_party/py/tensorflow_docs/api_generator/gen_java",
    ],
 )
@@ -0,0 +1,63 @@
 # Copyright 2022 The MediaPipe Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
 """Generate Java reference docs for MediaPipe."""
 import pathlib
 from absl import app
 from absl import flags
 from tensorflow_docs.api_generator import gen_java
 _OUT_DIR = flags.DEFINE_string('output_dir', '/tmp/mp_java/',
                               'Write docs here.')
 _SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api_docs/java',
                                 'Path prefix in the _toc.yaml')
 _ = flags.DEFINE_string('code_url_prefix', None,
                        '[UNUSED] The url prefix for links to code.')
 _ = flags.DEFINE_bool(
    'search_hints', True,
    '[UNUSED] Include metadata search hints in the generated files')
 _ANDROID_SDK = pathlib.Path('android/sdk/api/26.txt')
 def main(_) -> None:
  # Default to using a relative path to find the Java source.
  mp_root = pathlib.Path(__file__)
  while (mp_root := mp_root.parent).name != 'mediapipe':
    # Find the nearest `mediapipe` dir.
    pass
  # Find the root from which all packages are relative.
  root = mp_root.parent
  # Externally, parts of the repo are nested inside a mediapipe/ directory
  # that does not exist internally. Support both.
  if (mp_root / 'mediapipe').exists():
    mp_root = mp_root / 'mediapipe'
  gen_java.gen_java_docs(
      package='com.google.mediapipe',
      source_path=mp_root / 'tasks/java',
      output_dir=pathlib.Path(_OUT_DIR.value),
      site_path=pathlib.Path(_SITE_PATH.value),
      federated_docs={'https://developer.android.com': root / _ANDROID_SDK})
 if __name__ == '__main__':
  app.run(main)
@@ -0,0 +1,81 @@
 # Copyright 2022 The MediaPipe Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
 r"""MediaPipe Model Maker reference docs generation script.
 This script generates API reference docs for the `mediapipe` PIP package.
 $> pip install -U git+https://github.com/tensorflow/docs mediapipe-model-maker
 $> python build_model_maker_api_docs.py
 """
 import os
 from absl import app
 from absl import flags
 from tensorflow_docs.api_generator import generate_lib
 try:
  # mediapipe has not been set up to work with bazel yet, so catch & report.
  import mediapipe_model_maker  # pytype: disable=import-error
 except ImportError as e:
  raise ImportError('Please `pip install mediapipe-model-maker`.') from e
 PROJECT_SHORT_NAME = 'mediapipe_model_maker'
 PROJECT_FULL_NAME = 'MediaPipe Model Maker'
 _OUTPUT_DIR = flags.DEFINE_string(
    'output_dir',
    default='/tmp/generated_docs',
    help='Where to write the resulting docs.')
 _URL_PREFIX = flags.DEFINE_string(
    'code_url_prefix',
    'https://github.com/google/mediapipe/tree/master/mediapipe/model_maker',
    'The url prefix for links to code.')
 _SEARCH_HINTS = flags.DEFINE_bool(
    'search_hints', True,
    'Include metadata search hints in the generated files')
 _SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api_docs/python',
                                 'Path prefix in the _toc.yaml')
 def gen_api_docs():
  """Generates API docs for the mediapipe-model-maker package."""
  doc_generator = generate_lib.DocGenerator(
      root_title=PROJECT_FULL_NAME,
      py_modules=[(PROJECT_SHORT_NAME, mediapipe_model_maker)],
      base_dir=os.path.dirname(mediapipe_model_maker.__file__),
      code_url_prefix=_URL_PREFIX.value,
      search_hints=_SEARCH_HINTS.value,
      site_path=_SITE_PATH.value,
      callbacks=[],
  )
  doc_generator.build(_OUTPUT_DIR.value)
  print('Docs output to:', _OUTPUT_DIR.value)
 def main(_):
  gen_api_docs()
 if __name__ == '__main__':
  app.run(main)
@@ -0,0 +1,83 @@
 # Copyright 2022 The MediaPipe Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
 r"""MediaPipe reference docs generation script.
 This script generates API reference docs for the `mediapipe` PIP package.
 $> pip install -U git+https://github.com/tensorflow/docs mediapipe
 $> python build_py_api_docs.py
 """
 import os
 from absl import app
 from absl import flags
 from tensorflow_docs.api_generator import generate_lib
 try:
  # mediapipe has not been set up to work with bazel yet, so catch & report.
  import mediapipe as mp  # pytype: disable=import-error
 except ImportError as e:
  raise ImportError('Please `pip install mediapipe`.') from e
 PROJECT_SHORT_NAME = 'mp'
 PROJECT_FULL_NAME = 'MediaPipe'
 _OUTPUT_DIR = flags.DEFINE_string(
    'output_dir',
    default='/tmp/generated_docs',
    help='Where to write the resulting docs.')
 _URL_PREFIX = flags.DEFINE_string(
    'code_url_prefix',
    'https://github.com/google/mediapipe/blob/master/mediapipe',
    'The url prefix for links to code.')
 _SEARCH_HINTS = flags.DEFINE_bool(
    'search_hints', True,
    'Include metadata search hints in the generated files')
 _SITE_PATH = flags.DEFINE_string('site_path', '/mediapipe/api/solutions/python',
                                 'Path prefix in the _toc.yaml')
 def gen_api_docs():
  """Generates API docs for the mediapipe package."""
  if hasattr(mp, 'solutions'):
    del mp.solutions
  doc_generator = generate_lib.DocGenerator(
      root_title=PROJECT_FULL_NAME,
      py_modules=[(PROJECT_SHORT_NAME, mp)],
      base_dir=os.path.dirname(mp.__file__),
      code_url_prefix=_URL_PREFIX.value,
      search_hints=_SEARCH_HINTS.value,
      site_path=_SITE_PATH.value,
      callbacks=[],
  )
  doc_generator.build(_OUTPUT_DIR.value)
  print('Docs output to:', _OUTPUT_DIR.value)
 def main(_):
  gen_api_docs()
 if __name__ == '__main__':
  app.run(main)
@@ -143,6 +143,98 @@ Below is an example of how to create a subgraph named `TwoPassThroughSubgraph`.
    }
    ```
 ## Graph Options
 It is possible to specify a "graph options" protobuf for a MediaPipe graph
 similar to the [`Calculator Options`](calculators.md#calculator-options)
 protobuf specified for a MediaPipe calculator. These "graph options" can be
 specified where a graph is invoked, and used to populate calculator options and
 subgraph options within the graph.
 In a CalculatorGraphConfig, graph options can be specified for a subgraph
 exactly like calculator options, as shown below:
 ```
 node {
  calculator: "FlowLimiterCalculator"
  input_stream: "image"
  output_stream: "throttled_image"
  node_options: {
    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions] {
      max_in_flight: 1
    }
  }
 }
 node {
  calculator: "FaceDetectionSubgraph"
  input_stream: "IMAGE:throttled_image"
  node_options: {
    [type.googleapis.com/mediapipe.FaceDetectionOptions] {
      tensor_width: 192
      tensor_height: 192
    }
  }
 }
 ```
 In a CalculatorGraphConfig, graph options can be accepted and used to populate
 calculator options, as shown below:
 ```
 graph_options: {
  [type.googleapis.com/mediapipe.FaceDetectionOptions] {}
 }
 node: {
  calculator: "ImageToTensorCalculator"
  input_stream: "IMAGE:multi_backend_image"
  node_options: {
    [type.googleapis.com/mediapipe.ImageToTensorCalculatorOptions] {
        keep_aspect_ratio: true
        border_mode: BORDER_ZERO
    }
  }
  option_value: "output_tensor_width:options/tensor_width"
  option_value: "output_tensor_height:options/tensor_height"
 }
 node {
  calculator: "InferenceCalculator"
  node_options: {
    [type.googleapis.com/mediapipe.InferenceCalculatorOptions] {}
  }
  option_value: "delegate:options/delegate"
  option_value: "model_path:options/model_path"
 }
 ```
 In this example, the `FaceDetectionSubgraph` accepts graph option protobuf
 `FaceDetectionOptions`. The `FaceDetectionOptions` is used to define some field
 values in the calculator options `ImageToTensorCalculatorOptions` and some field
 values in the subgraph options `InferenceCalculatorOptions`. The field values
 are defined using the `option_value:` syntax.
 In the `CalculatorGraphConfig::Node` protobuf, the fields `node_options:` and
 `option_value:` together define the option values for a calculator such as
 `ImageToTensorCalculator`. The `node_options:` field defines a set of literal
 constant values using the text protobuf syntax. Each `option_value:` field
 defines the value for one protobuf field using information from the enclosing
 graph, specifically from field values of the graph options of the enclosing
 graph. In the example above, the `option_value:`
 `"output_tensor_width:options/tensor_width"` defines the field
 `ImageToTensorCalculatorOptions.output_tensor_width` using the value of
 `FaceDetectionOptions.tensor_width`.
 The syntax of `option_value:` is similar to the syntax of `input_stream:`. The
 syntax is `option_value: "LHS:RHS"`. The LHS identifies a calculator option
 field and the RHS identifies a graph option field. More specifically, the LHS
 and RHS each consists of a series of protobuf field names identifying nested
 protobuf messages and fields separated by '/'. This is known as the "ProtoPath"
 syntax. Nested messages that are referenced in the LHS or RHS must already be
 defined in the enclosing protobuf in order to be traversed using
 `option_value:`.
 ## Cycles
 <!-- TODO: add discussion of PreviousLoopbackCalculator -->
@@ -54,7 +54,7 @@ Note: This currently works only on Linux, and please first follow
    ```bash
    GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu \
-      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt
+      --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop_live_gpu.pbtxt
    ```
    This will open up your webcam as long as it is connected and on. Any errors
@@ -141,3 +141,4 @@ Nvidia Jetson and Raspberry Pi, please read
    ```bash
    (mp_env)mediapipe$ python3 setup.py bdist_wheel
    ```
 7. Exit from the MediaPipe repo directory and launch the Python interpreter.
@@ -259,6 +259,7 @@ mp_holistic = mp.solutions.holistic
 # For static images:
 IMAGE_FILES = []
 BG_COLOR = (192, 192, 192) # gray
 with mp_holistic.Holistic(
    static_image_mode=True,
    model_complexity=2,
@@ -94,8 +94,6 @@ one over the other.
 *   [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
 *   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
 *   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
 *   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
 ### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
@@ -217,7 +217,7 @@ A list of pose landmarks. Each landmark consists of the following:
 *Fig 5. Example of MediaPipe Pose real-world 3D coordinates.* |
 :-----------------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_world_landmarks.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/mobile/pose_world_landmarks.mp4" type="video/mp4"></video> |
 Another list of pose landmarks in world coordinates. Each landmark consists of
 the following:
@@ -238,7 +238,7 @@ for usage details.
 *Fig 6. Example of MediaPipe Pose segmentation mask.* |
 :---------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/mobile/pose_segmentation.mp4" type="video/mp4"></video> |
 ### Python Solution API
@@ -22,7 +22,7 @@ nav_order: 7
 *Fig 1. Example of MediaPipe Selfie Segmentation.* |
 :------------------------------------------------: |
-<video autoplay muted loop preload style="height: auto; width: 480px"><source src="../images/selfie_segmentation_web.mp4" type="video/mp4"></video> |
+<video autoplay muted loop preload style="height: auto; width: 480px"><source src="https://mediapipe.dev/images/selfie_segmentation_web.mp4" type="video/mp4"></video> |
 MediaPipe Selfie Segmentation segments the prominent humans in the scene. It can
 run in real-time on both smartphones and laptops. The intended use cases include
@@ -1,14 +1,13 @@
-"""Copyright 2019 - 2020 The MediaPipe Authors.
+# Copyright 2019 - 2022 The MediaPipe Authors.
-
+#
-Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+# You may obtain a copy of the License at
-
+#
-     http://www.apache.org/licenses/LICENSE-2.0
+#      http://www.apache.org/licenses/LICENSE-2.0
-
+#
-Unless required by applicable law or agreed to in writing, software
+# Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
+# distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
+# See the License for the specific language governing permissions and
-limitations under the License.
+# limitations under the License.
 """
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 licenses(["notice"])
 package(default_visibility = ["//visibility:private"])
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 proto_library(
    name = "mfcc_mel_calculators_proto",
    srcs = ["mfcc_mel_calculators.proto"],
@@ -197,7 +197,6 @@ cc_library(
        ":spectrogram_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:time_series_header_cc_proto",
        "//mediapipe/framework/port:core_proto",
        "//mediapipe/framework/port:integral_types",
        "//mediapipe/framework/port:logging",
@@ -277,6 +276,7 @@ cc_test(
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:status",
        "//mediapipe/util:time_series_test_util",
        "@com_google_audio_tools//audio/dsp/mfcc",
        "@eigen_archive//:eigen3",
    ],
 )
@@ -352,6 +352,8 @@ cc_test(
        "//mediapipe/framework/port:status",
        "//mediapipe/framework/tool:validate_type",
        "//mediapipe/util:time_series_test_util",
        "@com_google_audio_tools//audio/dsp:resampler",
        "@com_google_audio_tools//audio/dsp:resampler_q",
        "@com_google_audio_tools//audio/dsp:signal_vector_util",
        "@eigen_archive//:eigen3",
    ],
@@ -280,6 +280,13 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
                                                    &window);
      break;
    case SpectrogramCalculatorOptions::SQRT_HANN: {
      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
                                                 &window);
      absl::c_transform(window, window.begin(),
                        [](double x) { return std::sqrt(x); });
      break;
    }
  }
  // Propagate settings down to the actual Spectrogram object.
@@ -68,6 +68,7 @@ message SpectrogramCalculatorOptions {
    HANN = 0;
    HAMMING = 1;
    COSINE = 2;
    SQRT_HANN = 4;
  }
  optional WindowType window_type = 6 [default = HANN];
@@ -13,16 +13,24 @@
 # limitations under the License.
 #
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 licenses(["notice"])
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 selects.config_setting_group(
    name = "ios_or_disable_gpu",
    match_any = [
        "//mediapipe/gpu:disable_gpu",
        "//mediapipe:ios",
    ],
 )
 mediapipe_proto_library(
    name = "concatenate_vector_calculator_proto",
    srcs = ["concatenate_vector_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -32,7 +40,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "dequantize_byte_array_calculator_proto",
    srcs = ["dequantize_byte_array_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -42,7 +49,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "packet_cloner_calculator_proto",
    srcs = ["packet_cloner_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -52,7 +58,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "packet_resampler_calculator_proto",
    srcs = ["packet_resampler_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -62,7 +67,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "packet_thinner_calculator_proto",
    srcs = ["packet_thinner_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -72,7 +76,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "split_vector_calculator_proto",
    srcs = ["split_vector_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -82,7 +85,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "quantize_float_vector_calculator_proto",
    srcs = ["quantize_float_vector_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -92,7 +94,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "sequence_shift_calculator_proto",
    srcs = ["sequence_shift_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -102,7 +103,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "gate_calculator_proto",
    srcs = ["gate_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -112,7 +112,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "constant_side_packet_calculator_proto",
    srcs = ["constant_side_packet_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -124,7 +123,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "clip_vector_size_calculator_proto",
    srcs = ["clip_vector_size_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -134,7 +132,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "flow_limiter_calculator_proto",
    srcs = ["flow_limiter_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -144,7 +141,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "graph_profile_calculator_proto",
    srcs = ["graph_profile_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -154,7 +150,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "get_vector_item_calculator_proto",
    srcs = ["get_vector_item_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -164,7 +159,6 @@ mediapipe_proto_library(
 cc_library(
    name = "add_header_calculator",
    srcs = ["add_header_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -193,7 +187,6 @@ cc_library(
    name = "begin_loop_calculator",
    srcs = ["begin_loop_calculator.cc"],
    hdrs = ["begin_loop_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_contract",
@@ -216,16 +209,15 @@ cc_library(
    name = "end_loop_calculator",
    srcs = ["end_loop_calculator.cc"],
    hdrs = ["end_loop_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_contract",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:collection_item_id",
        "//mediapipe/framework:packet",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:detection_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework/port:integral_types",
        "//mediapipe/framework/port:ret_check",
@@ -258,7 +250,6 @@ cc_test(
 cc_library(
    name = "concatenate_vector_calculator_hdr",
    hdrs = ["concatenate_vector_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        ":concatenate_vector_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -284,7 +275,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":concatenate_vector_calculator_cc_proto",
        "//mediapipe/framework/api2:node",
@@ -311,7 +301,6 @@ cc_library(
 cc_library(
    name = "concatenate_detection_vector_calculator",
    srcs = ["concatenate_detection_vector_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":concatenate_vector_calculator",
        "//mediapipe/framework:calculator_framework",
@@ -323,11 +312,11 @@ cc_library(
 cc_library(
    name = "concatenate_proto_list_calculator",
    srcs = ["concatenate_proto_list_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":concatenate_vector_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
@@ -340,10 +329,10 @@ cc_test(
    srcs = ["concatenate_proto_list_calculator_test.cc"],
    deps = [
        ":concatenate_proto_list_calculator",
        ":concatenate_vector_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework:timestamp",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
@@ -371,7 +360,6 @@ cc_library(
    name = "clip_vector_size_calculator",
    srcs = ["clip_vector_size_calculator.cc"],
    hdrs = ["clip_vector_size_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        ":clip_vector_size_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -387,7 +375,6 @@ cc_library(
 cc_library(
    name = "clip_detection_vector_size_calculator",
    srcs = ["clip_detection_vector_size_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":clip_vector_size_calculator",
        "//mediapipe/framework:calculator_framework",
@@ -401,7 +388,6 @@ cc_test(
    srcs = ["clip_vector_size_calculator_test.cc"],
    deps = [
        ":clip_vector_size_calculator",
        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework:timestamp",
@@ -415,9 +401,6 @@ cc_test(
 cc_library(
    name = "counting_source_calculator",
    srcs = ["counting_source_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:ret_check",
@@ -430,9 +413,6 @@ cc_library(
 cc_library(
    name = "make_pair_calculator",
    srcs = ["make_pair_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -461,9 +441,6 @@ cc_test(
 cc_library(
    name = "matrix_multiply_calculator",
    srcs = ["matrix_multiply_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -477,9 +454,6 @@ cc_library(
 cc_library(
    name = "matrix_subtract_calculator",
    srcs = ["matrix_subtract_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -493,9 +467,6 @@ cc_library(
 cc_library(
    name = "mux_calculator",
    srcs = ["mux_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -508,9 +479,6 @@ cc_library(
 cc_library(
    name = "non_zero_calculator",
    srcs = ["non_zero_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -556,9 +524,6 @@ cc_test(
 cc_library(
    name = "packet_cloner_calculator",
    srcs = ["packet_cloner_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        ":packet_cloner_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -587,7 +552,6 @@ cc_test(
 cc_library(
    name = "packet_inner_join_calculator",
    srcs = ["packet_inner_join_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:ret_check",
@@ -611,9 +575,8 @@ cc_test(
 cc_library(
    name = "packet_thinner_calculator",
    srcs = ["packet_thinner_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
-        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        ":packet_thinner_calculator_cc_proto",
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:video_stream_header",
@@ -630,7 +593,7 @@ cc_test(
    srcs = ["packet_thinner_calculator_test.cc"],
    deps = [
        ":packet_thinner_calculator",
-        "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto",
+        ":packet_thinner_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework/formats:video_stream_header",
@@ -643,9 +606,6 @@ cc_test(
 cc_library(
    name = "pass_through_calculator",
    srcs = ["pass_through_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:status",
@@ -656,9 +616,6 @@ cc_library(
 cc_library(
    name = "round_robin_demux_calculator",
    srcs = ["round_robin_demux_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -670,9 +627,6 @@ cc_library(
 cc_library(
    name = "immediate_mux_calculator",
    srcs = ["immediate_mux_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:ret_check",
@@ -684,7 +638,6 @@ cc_library(
 cc_library(
    name = "packet_presence_calculator",
    srcs = ["packet_presence_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:packet",
@@ -713,7 +666,6 @@ cc_test(
 cc_library(
    name = "previous_loopback_calculator",
    srcs = ["previous_loopback_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:packet",
@@ -729,7 +681,6 @@ cc_library(
 cc_library(
    name = "flow_limiter_calculator",
    srcs = ["flow_limiter_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":flow_limiter_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -746,7 +697,6 @@ cc_library(
 cc_library(
    name = "string_to_int_calculator",
    srcs = ["string_to_int_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:integral_types",
@@ -759,7 +709,6 @@ cc_library(
 cc_library(
    name = "default_side_packet_calculator",
    srcs = ["default_side_packet_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:ret_check",
@@ -771,7 +720,6 @@ cc_library(
 cc_library(
    name = "side_packet_to_stream_calculator",
    srcs = ["side_packet_to_stream_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:logging",
@@ -822,11 +770,8 @@ cc_library(
    name = "packet_resampler_calculator",
    srcs = ["packet_resampler_calculator.cc"],
    hdrs = ["packet_resampler_calculator.h"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        ":packet_resampler_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:collection_item_id",
        "//mediapipe/framework/deps:mathutil",
@@ -850,7 +795,7 @@ cc_test(
    ],
    deps = [
        ":packet_resampler_calculator",
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        ":packet_resampler_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework/formats:video_stream_header",
@@ -884,7 +829,6 @@ cc_test(
 cc_test(
    name = "matrix_multiply_calculator_test",
    srcs = ["matrix_multiply_calculator_test.cc"],
    visibility = ["//visibility:private"],
    deps = [
        ":matrix_multiply_calculator",
        "//mediapipe/framework:calculator_framework",
@@ -900,7 +844,6 @@ cc_test(
 cc_test(
    name = "matrix_subtract_calculator_test",
    srcs = ["matrix_subtract_calculator_test.cc"],
    visibility = ["//visibility:private"],
    deps = [
        ":matrix_subtract_calculator",
        "//mediapipe/framework:calculator_framework",
@@ -918,10 +861,10 @@ cc_test(
    name = "flow_limiter_calculator_test",
    srcs = ["flow_limiter_calculator_test.cc"],
    deps = [
        ":counting_source_calculator",
        ":flow_limiter_calculator",
        ":flow_limiter_calculator_cc_proto",
-        "//mediapipe/calculators/core:counting_source_calculator",
+        ":pass_through_calculator",
        "//mediapipe/calculators/core:pass_through_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework:test_calculators",
@@ -934,6 +877,7 @@ cc_test(
        "//mediapipe/framework/tool:simulation_clock",
        "//mediapipe/framework/tool:simulation_clock_executor",
        "//mediapipe/framework/tool:sink",
        "//mediapipe/util:packet_test_util",
        "@com_google_absl//absl/time",
    ],
 )
@@ -949,14 +893,13 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":split_vector_calculator_cc_proto",
        "//mediapipe/framework/formats:detection_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
@@ -965,8 +908,7 @@ cc_library(
        "@org_tensorflow//tensorflow/lite:framework",
        "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
    ] + select({
-        "//mediapipe/gpu:disable_gpu": [],
+        ":ios_or_disable_gpu": [],
        "//mediapipe:ios": [],
        "//conditions:default": [
            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
        ],
@@ -995,7 +937,6 @@ cc_test(
 cc_library(
    name = "split_proto_list_calculator",
    srcs = ["split_proto_list_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":split_vector_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -1027,7 +968,6 @@ cc_test(
 cc_library(
    name = "dequantize_byte_array_calculator",
    srcs = ["dequantize_byte_array_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":dequantize_byte_array_calculator_cc_proto",
        "//mediapipe/framework:calculator_context",
@@ -1053,7 +993,6 @@ cc_test(
 cc_library(
    name = "quantize_float_vector_calculator",
    srcs = ["quantize_float_vector_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":quantize_float_vector_calculator_cc_proto",
        "//mediapipe/framework:calculator_context",
@@ -1079,7 +1018,6 @@ cc_test(
 cc_library(
    name = "sequence_shift_calculator",
    srcs = ["sequence_shift_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":sequence_shift_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -1104,7 +1042,6 @@ cc_test(
 cc_library(
    name = "gate_calculator",
    srcs = ["gate_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":gate_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -1130,7 +1067,6 @@ cc_test(
 cc_library(
    name = "matrix_to_vector_calculator",
    srcs = ["matrix_to_vector_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -1166,7 +1102,6 @@ cc_test(
 cc_library(
    name = "merge_calculator",
    srcs = ["merge_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -1192,7 +1127,6 @@ cc_test(
 cc_library(
    name = "stream_to_side_packet_calculator",
    srcs = ["stream_to_side_packet_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:timestamp",
@@ -1218,7 +1152,6 @@ cc_test(
 cc_library(
    name = "constant_side_packet_calculator",
    srcs = ["constant_side_packet_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":constant_side_packet_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -1248,7 +1181,6 @@ cc_test(
 cc_library(
    name = "graph_profile_calculator",
    srcs = ["graph_profile_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":graph_profile_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -1290,14 +1222,14 @@ cc_library(
    name = "get_vector_item_calculator",
    srcs = ["get_vector_item_calculator.cc"],
    hdrs = ["get_vector_item_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        ":get_vector_item_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:packet",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:packet",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:detection_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
@@ -1319,11 +1251,35 @@ cc_test(
    ],
 )
 cc_library(
    name = "vector_indices_calculator",
    srcs = ["vector_indices_calculator.cc"],
    hdrs = ["vector_indices_calculator.h"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/port:status",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "vector_indices_calculator_test",
    srcs = ["vector_indices_calculator_test.cc"],
    deps = [
        ":vector_indices_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
    ],
 )
 cc_library(
    name = "vector_size_calculator",
    srcs = ["vector_size_calculator.cc"],
    hdrs = ["vector_size_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -1337,9 +1293,6 @@ cc_library(
 cc_library(
    name = "packet_sequencer_calculator",
    srcs = ["packet_sequencer_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:contract",
@@ -1357,7 +1310,7 @@ cc_test(
    srcs = ["packet_sequencer_calculator_test.cc"],
    deps = [
        ":packet_sequencer_calculator",
-        "//mediapipe/calculators/core:pass_through_calculator",
+        ":pass_through_calculator",
        "//mediapipe/framework:calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:subgraph",
@@ -1374,13 +1327,53 @@ cc_library(
    name = "merge_to_vector_calculator",
    srcs = ["merge_to_vector_calculator.cc"],
    hdrs = ["merge_to_vector_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:detection_cc_proto",
        "//mediapipe/framework/formats:image",
        "@com_google_absl//absl/status",
    ],
    alwayslink = 1,
 )
 mediapipe_proto_library(
    name = "bypass_calculator_proto",
    srcs = ["bypass_calculator.proto"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
    ],
 )
 cc_library(
    name = "bypass_calculator",
    srcs = ["bypass_calculator.cc"],
    deps = [
        ":bypass_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:collection_item_id",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/port:status",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "bypass_calculator_test",
    srcs = ["bypass_calculator_test.cc"],
    deps = [
        ":bypass_calculator",
        ":pass_through_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "//mediapipe/framework/tool:switch_container",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/types:optional",
    ],
 )
@@ -0,0 +1,165 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <algorithm>
 #include <map>
 #include <memory>
 #include <set>
 #include <string>
 #include <utility>
 #include <vector>
 #include "mediapipe/calculators/core/bypass_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/port/status.h"
 namespace mediapipe {
 namespace api2 {
 using mediapipe::BypassCalculatorOptions;
 // Defines a "bypass" channel to use in place of a disabled feature subgraph.
 // By default, all inputs are discarded and all outputs are ignored.
 // Certain input streams can be passed to corresponding output streams
 // by specifying them in "pass_input_stream" and "pass_output_stream" options.
 // All output streams are updated with timestamp bounds indicating completed
 // output.
 //
 // Note that this calculator is designed for use as a contained_node in a
 // SwitchContainer.  For this reason, any input and output tags are accepted,
 // and stream semantics are specified through BypassCalculatorOptions.
 //
 // Example config:
 //     node {
 //       calculator: "BypassCalculator"
 //       input_stream: "APPEARANCES:appearances_post_facenet"
 //       input_stream: "VIDEO:video_frame"
 //       input_stream: "FEATURE_CONFIG:feature_config"
 //       input_stream: "ENABLE:gaze_enabled"
 //       output_stream: "APPEARANCES:analyzed_appearances"
 //       output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
 //       node_options: {
 //         [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
 //           pass_input_stream: "APPEARANCES"
 //           pass_output_stream: "APPEARANCES"
 //         }
 //       }
 //     }
 //
 class BypassCalculator : public Node {
 public:
  static constexpr mediapipe::api2::Input<int>::Optional kNotNeeded{"N_N_"};
  MEDIAPIPE_NODE_CONTRACT(kNotNeeded);
  using IdMap = std::map<CollectionItemId, CollectionItemId>;
  // Returns the map of passthrough input and output stream ids.
  static absl::StatusOr<IdMap> GetPassMap(
      const BypassCalculatorOptions& options, const tool::TagMap& input_map,
      const tool::TagMap& output_map) {
    IdMap result;
    auto& input_streams = options.pass_input_stream();
    auto& output_streams = options.pass_output_stream();
    int size = std::min(input_streams.size(), output_streams.size());
    for (int i = 0; i < size; ++i) {
      std::pair<std::string, int> in_tag, out_tag;
      MP_RETURN_IF_ERROR(tool::ParseTagIndex(options.pass_input_stream(i),
                                             &in_tag.first, &in_tag.second));
      MP_RETURN_IF_ERROR(tool::ParseTagIndex(options.pass_output_stream(i),
                                             &out_tag.first, &out_tag.second));
      auto input_id = input_map.GetId(in_tag.first, in_tag.second);
      auto output_id = output_map.GetId(out_tag.first, out_tag.second);
      result[input_id] = output_id;
    }
    return result;
  }
  // Identifies all specified streams as "Any" packet type.
  // Identifies passthrough streams as "Same" packet type.
  static absl::Status UpdateContract(CalculatorContract* cc) {
    auto options = cc->Options<BypassCalculatorOptions>();
    RET_CHECK_EQ(options.pass_input_stream().size(),
                 options.pass_output_stream().size());
    ASSIGN_OR_RETURN(
        auto pass_streams,
        GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
    std::set<CollectionItemId> pass_out;
    for (auto entry : pass_streams) {
      pass_out.insert(entry.second);
      cc->Inputs().Get(entry.first).SetAny();
      cc->Outputs().Get(entry.second).SetSameAs(&cc->Inputs().Get(entry.first));
    }
    for (auto id = cc->Inputs().BeginId(); id != cc->Inputs().EndId(); ++id) {
      if (pass_streams.count(id) == 0) {
        cc->Inputs().Get(id).SetAny();
      }
    }
    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
      if (pass_out.count(id) == 0) {
        cc->Outputs().Get(id).SetAny();
      }
    }
    for (auto id = cc->InputSidePackets().BeginId();
         id != cc->InputSidePackets().EndId(); ++id) {
      cc->InputSidePackets().Get(id).SetAny();
    }
    return absl::OkStatus();
  }
  // Saves the map of passthrough input and output stream ids.
  absl::Status Open(CalculatorContext* cc) override {
    auto options = cc->Options<BypassCalculatorOptions>();
    ASSIGN_OR_RETURN(pass_streams_, GetPassMap(options, *cc->Inputs().TagMap(),
                                               *cc->Outputs().TagMap()));
    return absl::OkStatus();
  }
  // Copies packets between passthrough input and output streams.
  // Updates timestamp bounds on all output streams.
  absl::Status Process(CalculatorContext* cc) override {
    std::set<CollectionItemId> pass_out;
    for (auto entry : pass_streams_) {
      pass_out.insert(entry.second);
      auto& packet = cc->Inputs().Get(entry.first).Value();
      if (packet.Timestamp() == cc->InputTimestamp()) {
        cc->Outputs().Get(entry.second).AddPacket(packet);
      }
    }
    Timestamp bound = cc->InputTimestamp().NextAllowedInStream();
    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
      if (pass_out.count(id) == 0) {
        cc->Outputs().Get(id).SetNextTimestampBound(
            std::max(cc->Outputs().Get(id).NextTimestampBound(), bound));
      }
    }
    return absl::OkStatus();
  }
  // Close all output streams.
  absl::Status Close(CalculatorContext* cc) override {
    for (auto id = cc->Outputs().BeginId(); id != cc->Outputs().EndId(); ++id) {
      cc->Outputs().Get(id).Close();
    }
    return absl::OkStatus();
  }
 private:
  IdMap pass_streams_;
 };
 MEDIAPIPE_REGISTER_NODE(BypassCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,31 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 message BypassCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional BypassCalculatorOptions ext = 481259677;
  }
  // Names an input stream or streams to pass through, by "TAG:index".
  repeated string pass_input_stream = 1;
  // Names an output stream or streams to pass through, by "TAG:index".
  repeated string pass_output_stream = 2;
 }
@@ -0,0 +1,304 @@
 // Copyright 2019 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <memory>
 #include <string>
 #include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/optional.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_matchers.h"
 namespace mediapipe {
 namespace {
 // A graph with using a BypassCalculator to pass through and ignore
 // most of its inputs and outputs.
 constexpr char kTestGraphConfig1[] = R"pb(
  type: "AppearancesPassThroughSubgraph"
  input_stream: "APPEARANCES:appearances"
  input_stream: "VIDEO:video_frame"
  input_stream: "FEATURE_CONFIG:feature_config"
  output_stream: "APPEARANCES:passthrough_appearances"
  output_stream: "FEDERATED_GAZE_OUTPUT:passthrough_federated_gaze_output"
  node {
    calculator: "BypassCalculator"
    input_stream: "PASS:appearances"
    input_stream: "IGNORE:0:video_frame"
    input_stream: "IGNORE:1:feature_config"
    output_stream: "PASS:passthrough_appearances"
    output_stream: "IGNORE:passthrough_federated_gaze_output"
    node_options: {
      [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
        pass_input_stream: "PASS"
        pass_output_stream: "PASS"
      }
    }
  }
 )pb";
 // A graph with using AppearancesPassThroughSubgraph as a do-nothing channel
 // for input frames and appearances.
 constexpr char kTestGraphConfig2[] = R"pb(
  input_stream: "VIDEO_FULL_RES:video_frame"
  input_stream: "APPEARANCES:input_appearances"
  input_stream: "FEATURE_CONFIG:feature_config"
  input_stream: "GAZE_ENABLED:gaze_enabled"
  output_stream: "APPEARANCES:analyzed_appearances"
  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
  node {
    calculator: "SwitchContainer"
    input_stream: "VIDEO:video_frame"
    input_stream: "APPEARANCES:input_appearances"
    input_stream: "FEATURE_CONFIG:feature_config"
    input_stream: "ENABLE:gaze_enabled"
    output_stream: "APPEARANCES:analyzed_appearances"
    output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
    options {
      [mediapipe.SwitchContainerOptions.ext] {
        async_selection: true
        contained_node: { calculator: "AppearancesPassThroughSubgraph" }
      }
    }
  }
 )pb";
 // A graph with using BypassCalculator as a do-nothing channel
 // for input frames and appearances.
 constexpr char kTestGraphConfig3[] = R"pb(
  input_stream: "VIDEO_FULL_RES:video_frame"
  input_stream: "APPEARANCES:input_appearances"
  input_stream: "FEATURE_CONFIG:feature_config"
  input_stream: "GAZE_ENABLED:gaze_enabled"
  output_stream: "APPEARANCES:analyzed_appearances"
  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
  node {
    calculator: "SwitchContainer"
    input_stream: "VIDEO:video_frame"
    input_stream: "APPEARANCES:input_appearances"
    input_stream: "FEATURE_CONFIG:feature_config"
    input_stream: "ENABLE:gaze_enabled"
    output_stream: "APPEARANCES:analyzed_appearances"
    output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
    options {
      [mediapipe.SwitchContainerOptions.ext] {
        async_selection: true
        contained_node: {
          calculator: "BypassCalculator"
          node_options: {
            [type.googleapis.com/mediapipe.BypassCalculatorOptions] {
              pass_input_stream: "APPEARANCES"
              pass_output_stream: "APPEARANCES"
            }
          }
        }
      }
    }
  }
 )pb";
 // A graph with using BypassCalculator as a disabled-gate
 // for input frames and appearances.
 constexpr char kTestGraphConfig4[] = R"pb(
  input_stream: "VIDEO_FULL_RES:video_frame"
  input_stream: "APPEARANCES:input_appearances"
  input_stream: "FEATURE_CONFIG:feature_config"
  input_stream: "GAZE_ENABLED:gaze_enabled"
  output_stream: "APPEARANCES:analyzed_appearances"
  output_stream: "FEDERATED_GAZE_OUTPUT:federated_gaze_output"
  node {
    calculator: "SwitchContainer"
    input_stream: "ENABLE:gaze_enabled"
    input_stream: "VIDEO:video_frame"
    input_stream: "APPEARANCES:input_appearances"
    input_stream: "FEATURE_CONFIG:feature_config"
    output_stream: "VIDEO:video_frame_out"
    output_stream: "APPEARANCES:analyzed_appearances"
    output_stream: "FEATURE_CONFIG:feature_config_out"
    options {
      [mediapipe.SwitchContainerOptions.ext] {
        contained_node: { calculator: "BypassCalculator" }
        contained_node: { calculator: "PassThroughCalculator" }
      }
    }
  }
 )pb";
 // Reports packet timestamp and string contents, or "<empty>"".
 std::string DebugString(Packet p) {
  return absl::StrCat(p.Timestamp().DebugString(), ":",
                      p.IsEmpty() ? "<empty>" : p.Get<std::string>());
 }
 // Shows a bypass subgraph that passes through one stream.
 TEST(BypassCalculatorTest, SubgraphChannel) {
  CalculatorGraphConfig config_1 =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig1);
  CalculatorGraphConfig config_2 =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig2);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize({config_1, config_2}, {}));
  std::vector<std::string> analyzed_appearances;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "analyzed_appearances",
      [&](const Packet& p) {
        analyzed_appearances.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  std::vector<std::string> federated_gaze_output;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "federated_gaze_output",
      [&](const Packet& p) {
        federated_gaze_output.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  MP_ASSERT_OK(graph.StartRun({}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:a1"));
  EXPECT_THAT(federated_gaze_output, testing::ElementsAre("200:<empty>"));
  MP_ASSERT_OK(graph.CloseAllInputStreams());
  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 // Shows a BypassCalculator that passes through one stream.
 TEST(BypassCalculatorTest, CalculatorChannel) {
  CalculatorGraphConfig config_3 =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig3);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize({config_3}, {}));
  std::vector<std::string> analyzed_appearances;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "analyzed_appearances",
      [&](const Packet& p) {
        analyzed_appearances.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  std::vector<std::string> federated_gaze_output;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "federated_gaze_output",
      [&](const Packet& p) {
        federated_gaze_output.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  MP_ASSERT_OK(graph.StartRun({}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:a1"));
  EXPECT_THAT(federated_gaze_output, testing::ElementsAre("200:<empty>"));
  MP_ASSERT_OK(graph.CloseAllInputStreams());
  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 // Shows a BypassCalculator that discards all inputs when ENABLED is false.
 TEST(BypassCalculatorTest, GatedChannel) {
  CalculatorGraphConfig config_3 =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(kTestGraphConfig4);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize({config_3}, {}));
  std::vector<std::string> analyzed_appearances;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "analyzed_appearances",
      [&](const Packet& p) {
        analyzed_appearances.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  std::vector<std::string> video_frame;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "video_frame_out",
      [&](const Packet& p) {
        video_frame.push_back(DebugString(p));
        return absl::OkStatus();
      },
      true));
  MP_ASSERT_OK(graph.StartRun({}));
  // Close the gate.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "gaze_enabled", MakePacket<bool>(false).At(Timestamp(200))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  // Send packets at timestamp 200.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input_appearances", MakePacket<std::string>("a1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "video_frame", MakePacket<std::string>("v1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feature_config", MakePacket<std::string>("f1").At(Timestamp(200))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  // Only timestamps arrive from the BypassCalculator.
  EXPECT_THAT(analyzed_appearances, testing::ElementsAre("200:<empty>"));
  EXPECT_THAT(video_frame, testing::ElementsAre("200:<empty>"));
  // Open the gate.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "gaze_enabled", MakePacket<bool>(true).At(Timestamp(300))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  // Send packets at timestamp 300.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input_appearances", MakePacket<std::string>("a2").At(Timestamp(300))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "video_frame", MakePacket<std::string>("v2").At(Timestamp(300))));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feature_config", MakePacket<std::string>("f2").At(Timestamp(300))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  // Packets arrive from the PassThroughCalculator.
  EXPECT_THAT(analyzed_appearances,
              testing::ElementsAre("200:<empty>", "300:a2"));
  EXPECT_THAT(video_frame, testing::ElementsAre("200:<empty>", "300:v2"));
  MP_ASSERT_OK(graph.CloseAllInputStreams());
  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 }  // namespace
 }  // namespace mediapipe
@@ -18,6 +18,7 @@
 #include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -111,6 +112,22 @@ class ConcatenateLandmarkListCalculator
 };
 MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkListCalculator);
 class ConcatenateClassificationListCalculator
    : public ConcatenateListsCalculator<Classification, ClassificationList> {
 protected:
  int ListSize(const ClassificationList& list) const override {
    return list.classification_size();
  }
  const Classification GetItem(const ClassificationList& list,
                               int idx) const override {
    return list.classification(idx);
  }
  Classification* AddItem(ClassificationList& list) const override {
    return list.add_classification();
  }
 };
 MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -18,6 +18,7 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -70,6 +71,16 @@ void AddInputLandmarkLists(
  }
 }
 void AddInputClassificationLists(
    const std::vector<ClassificationList>& input_classifications_vec,
    int64 timestamp, CalculatorRunner* runner) {
  for (int i = 0; i < input_classifications_vec.size(); ++i) {
    runner->MutableInputs()->Index(i).packets.push_back(
        MakePacket<ClassificationList>(input_classifications_vec[i])
            .At(Timestamp(timestamp)));
  }
 }
 TEST(ConcatenateNormalizedLandmarkListCalculatorTest, EmptyVectorInputs) {
  CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator",
                          /*options_string=*/"", /*num_inputs=*/3,
@@ -181,4 +192,39 @@ TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneEmptyStreamNoOutput) {
  EXPECT_EQ(0, outputs.size());
 }
 TEST(ConcatenateClassificationListCalculatorTest, OneTimestamp) {
  CalculatorRunner runner("ConcatenateClassificationListCalculator",
                          /*options_string=*/
                          "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
                          "{only_emit_if_all_present: true}",
                          /*num_inputs=*/2,
                          /*num_outputs=*/1, /*num_side_packets=*/0);
  auto input_0 = ParseTextProtoOrDie<ClassificationList>(R"pb(
    classification: { index: 0 score: 0.2 label: "test_0" }
    classification: { index: 1 score: 0.3 label: "test_1" }
    classification: { index: 2 score: 0.4 label: "test_2" }
  )pb");
  auto input_1 = ParseTextProtoOrDie<ClassificationList>(R"pb(
    classification: { index: 3 score: 0.2 label: "test_3" }
    classification: { index: 4 score: 0.3 label: "test_4" }
  )pb");
  std::vector<ClassificationList> inputs = {input_0, input_1};
  AddInputClassificationLists(inputs, /*timestamp=*/1, &runner);
  MP_ASSERT_OK(runner.Run());
  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
  EXPECT_EQ(1, outputs.size());
  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
  auto result = outputs[0].Get<ClassificationList>();
  EXPECT_THAT(ParseTextProtoOrDie<ClassificationList>(R"pb(
                classification: { index: 0 score: 0.2 label: "test_0" }
                classification: { index: 1 score: 0.3 label: "test_1" }
                classification: { index: 2 score: 0.4 label: "test_2" }
                classification: { index: 3 score: 0.2 label: "test_3" }
                classification: { index: 4 score: 0.3 label: "test_4" }
              )pb"),
              EqualsProto(result));
 }
 }  // namespace mediapipe
@@ -19,6 +19,7 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
 #include "tensorflow/lite/interpreter.h"
@@ -40,6 +41,9 @@ REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator);
 typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
 REGISTER_CALCULATOR(EndLoopBooleanCalculator);
 typedef EndLoopCalculator<std::vector<float>> EndLoopFloatCalculator;
 REGISTER_CALCULATOR(EndLoopFloatCalculator);
 typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
    EndLoopRenderDataCalculator;
 REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
@@ -55,4 +59,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
    EndLoopDetectionCalculator;
 REGISTER_CALCULATOR(EndLoopDetectionCalculator);
 typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator;
 REGISTER_CALCULATOR(EndLoopMatrixCalculator);
 }  // namespace mediapipe
@@ -50,7 +50,7 @@ namespace mediapipe {
 //   calculator:    "EndLoopWithOutputCalculator"
 //   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
 //   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
 // }
 template <typename IterableT>
 class EndLoopCalculator : public CalculatorBase {
@@ -18,7 +18,6 @@
 #include "mediapipe/calculators/core/flow_limiter_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/header_util.h"
@@ -68,7 +67,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
 // FlowLimiterCalculator provides limited support for multiple input streams.
 // The first input stream is treated as the main input stream and successive
 // input streams are treated as auxiliary input streams.  The auxiliary input
-// streams are limited to timestamps passed on the main input stream.
+// streams are limited to timestamps allowed by the "ALLOW" stream.
 //
 class FlowLimiterCalculator : public CalculatorBase {
 public:
@@ -100,64 +99,11 @@ class FlowLimiterCalculator : public CalculatorBase {
          cc->InputSidePackets().Tag(kMaxInFlightTag).Get<int>());
    }
    input_queues_.resize(cc->Inputs().NumEntries(""));
    allowed_[Timestamp::Unset()] = true;
    RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs())));
    return absl::OkStatus();
  }
  // Returns true if an additional frame can be released for processing.
  // The "ALLOW" output stream indicates this condition at each input frame.
  bool ProcessingAllowed() {
    return frames_in_flight_.size() < options_.max_in_flight();
  }
  // Outputs a packet indicating whether a frame was sent or dropped.
  void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
    if (cc->Outputs().HasTag(kAllowTag)) {
      cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
    }
  }
  // Sets the timestamp bound or closes an output stream.
  void SetNextTimestampBound(Timestamp bound, OutputStream* stream) {
    if (bound > Timestamp::Max()) {
      stream->Close();
    } else {
      stream->SetNextTimestampBound(bound);
    }
  }
  // Returns true if a certain timestamp is being processed.
  bool IsInFlight(Timestamp timestamp) {
    return std::find(frames_in_flight_.begin(), frames_in_flight_.end(),
                     timestamp) != frames_in_flight_.end();
  }
  // Releases input packets up to the latest settled input timestamp.
  void ProcessAuxiliaryInputs(CalculatorContext* cc) {
    Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound();
    for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) {
      // Release settled frames from each input queue.
      while (!input_queues_[i].empty() &&
             input_queues_[i].front().Timestamp() < settled_bound) {
        Packet packet = input_queues_[i].front();
        input_queues_[i].pop_front();
        if (IsInFlight(packet.Timestamp())) {
          cc->Outputs().Get("", i).AddPacket(packet);
        }
      }
      // Propagate each input timestamp bound.
      if (!input_queues_[i].empty()) {
        Timestamp bound = input_queues_[i].front().Timestamp();
        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
      } else {
        Timestamp bound =
            cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream();
        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
      }
    }
  }
  // Releases input packets allowed by the max_in_flight constraint.
  absl::Status Process(CalculatorContext* cc) final {
    options_ = tool::RetrieveOptions(options_, cc->Inputs());
@@ -224,13 +170,97 @@ class FlowLimiterCalculator : public CalculatorBase {
    }
    ProcessAuxiliaryInputs(cc);
    // Discard old ALLOW ranges.
    Timestamp input_bound = InputTimestampBound(cc);
    auto first_range = std::prev(allowed_.upper_bound(input_bound));
    allowed_.erase(allowed_.begin(), first_range);
    return absl::OkStatus();
  }
  int LedgerSize() {
    int result = frames_in_flight_.size() + allowed_.size();
    for (const auto& queue : input_queues_) {
      result += queue.size();
    }
    return result;
  }
 private:
  // Returns true if an additional frame can be released for processing.
  // The "ALLOW" output stream indicates this condition at each input frame.
  bool ProcessingAllowed() {
    return frames_in_flight_.size() < options_.max_in_flight();
  }
  // Outputs a packet indicating whether a frame was sent or dropped.
  void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) {
    if (cc->Outputs().HasTag(kAllowTag)) {
      cc->Outputs().Tag(kAllowTag).AddPacket(MakePacket<bool>(allow).At(ts));
    }
    allowed_[ts] = allow;
  }
  // Returns true if a timestamp falls within a range of allowed timestamps.
  bool IsAllowed(Timestamp timestamp) {
    auto it = allowed_.upper_bound(timestamp);
    return std::prev(it)->second;
  }
  // Sets the timestamp bound or closes an output stream.
  void SetNextTimestampBound(Timestamp bound, OutputStream* stream) {
    if (bound > Timestamp::Max()) {
      stream->Close();
    } else {
      stream->SetNextTimestampBound(bound);
    }
  }
  // Returns the lowest unprocessed input Timestamp.
  Timestamp InputTimestampBound(CalculatorContext* cc) {
    Timestamp result = Timestamp::Done();
    for (int i = 0; i < input_queues_.size(); ++i) {
      auto& queue = input_queues_[i];
      auto& stream = cc->Inputs().Get("", i);
      Timestamp bound = queue.empty()
                            ? stream.Value().Timestamp().NextAllowedInStream()
                            : queue.front().Timestamp();
      result = std::min(result, bound);
    }
    return result;
  }
  // Releases input packets up to the latest settled input timestamp.
  void ProcessAuxiliaryInputs(CalculatorContext* cc) {
    Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound();
    for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) {
      // Release settled frames from each input queue.
      while (!input_queues_[i].empty() &&
             input_queues_[i].front().Timestamp() < settled_bound) {
        Packet packet = input_queues_[i].front();
        input_queues_[i].pop_front();
        if (IsAllowed(packet.Timestamp())) {
          cc->Outputs().Get("", i).AddPacket(packet);
        }
      }
      // Propagate each input timestamp bound.
      if (!input_queues_[i].empty()) {
        Timestamp bound = input_queues_[i].front().Timestamp();
        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
      } else {
        Timestamp bound =
            cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream();
        SetNextTimestampBound(bound, &cc->Outputs().Get("", i));
      }
    }
  }
 private:
  FlowLimiterCalculatorOptions options_;
  std::vector<std::deque<Packet>> input_queues_;
  std::deque<Timestamp> frames_in_flight_;
  std::map<Timestamp, bool> allowed_;
 };
 REGISTER_CALCULATOR(FlowLimiterCalculator);
@@ -15,6 +15,7 @@
 #include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 #include "absl/time/clock.h"
@@ -32,6 +33,7 @@
 #include "mediapipe/framework/tool/simulation_clock.h"
 #include "mediapipe/framework/tool/simulation_clock_executor.h"
 #include "mediapipe/framework/tool/sink.h"
 #include "mediapipe/util/packet_test_util.h"
 namespace mediapipe {
@@ -77,6 +79,12 @@ std::vector<T> PacketValues(const std::vector<Packet>& packets) {
  return result;
 }
 std::string SourceString(Timestamp t) {
  return (t.IsSpecialValue())
             ? t.DebugString()
             : absl::StrCat("Timestamp(", t.DebugString(), ")");
 }
 // A Calculator::Process callback function.
 typedef std::function<absl::Status(const InputStreamShardSet&,
                                   OutputStreamShardSet*)>
@@ -651,11 +659,12 @@ TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) {
      input_packets_[17], input_packets_[19], input_packets_[20],
  };
  EXPECT_EQ(out_1_packets_, expected_output);
-  // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled.
+  // The timestamps released by FlowLimiterCalculator for in_1_sampled,
  // plus input_packets_[21].
  std::vector<Packet> expected_output_2 = {
      input_packets_[0],  input_packets_[2],  input_packets_[4],
      input_packets_[14], input_packets_[17], input_packets_[19],
-      input_packets_[20],
+      input_packets_[20], input_packets_[21],
  };
  EXPECT_EQ(out_2_packets, expected_output_2);
 }
@@ -699,11 +708,10 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
        }
      )pb");
-  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(R"pb(
+  auto limiter_options = ParseTextProtoOrDie<FlowLimiterCalculatorOptions>(
-    max_in_flight: 1
+      R"pb(
-    max_in_queue: 0
+        max_in_flight: 1 max_in_queue: 0 in_flight_timeout: 100000  # 100 ms
-    in_flight_timeout: 100000  # 100 ms
+      )pb");
  )pb");
  std::map<std::string, Packet> side_packets = {
      {"limiter_options",
       MakePacket<FlowLimiterCalculatorOptions>(limiter_options)},
@@ -759,13 +767,217 @@ TEST_F(FlowLimiterCalculatorTest, ZeroQueue) {
      input_packets_[0],  input_packets_[2],  input_packets_[15],
      input_packets_[17], input_packets_[19],
  };
-  EXPECT_EQ(out_1_packets_, expected_output);
+  EXPECT_THAT(out_1_packets_,
              ElementsAreArray(PacketMatchers<int>(expected_output)));
  // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled.
  std::vector<Packet> expected_output_2 = {
      input_packets_[0],  input_packets_[2],  input_packets_[4],
      input_packets_[15], input_packets_[17], input_packets_[19],
  };
-  EXPECT_EQ(out_2_packets, expected_output_2);
+  EXPECT_THAT(out_2_packets,
              ElementsAreArray(PacketMatchers<int>(expected_output_2)));
  // Validate the ALLOW stream output.
  std::vector<Packet> expected_allow = {
      MakePacket<bool>(true).At(Timestamp(0)),
      MakePacket<bool>(false).At(Timestamp(10000)),
      MakePacket<bool>(true).At(Timestamp(20000)),
      MakePacket<bool>(false).At(Timestamp(30000)),
      MakePacket<bool>(true).At(Timestamp(40000)),
      MakePacket<bool>(false).At(Timestamp(50000)),
      MakePacket<bool>(false).At(Timestamp(60000)),
      MakePacket<bool>(false).At(Timestamp(70000)),
      MakePacket<bool>(false).At(Timestamp(80000)),
      MakePacket<bool>(false).At(Timestamp(90000)),
      MakePacket<bool>(false).At(Timestamp(100000)),
      MakePacket<bool>(false).At(Timestamp(110000)),
      MakePacket<bool>(false).At(Timestamp(120000)),
      MakePacket<bool>(false).At(Timestamp(130000)),
      MakePacket<bool>(false).At(Timestamp(140000)),
      MakePacket<bool>(true).At(Timestamp(150000)),
      MakePacket<bool>(false).At(Timestamp(160000)),
      MakePacket<bool>(true).At(Timestamp(170000)),
      MakePacket<bool>(false).At(Timestamp(180000)),
      MakePacket<bool>(true).At(Timestamp(190000)),
      MakePacket<bool>(false).At(Timestamp(200000)),
  };
  EXPECT_THAT(allow_packets_,
              ElementsAreArray(PacketMatchers<bool>(expected_allow)));
 }
 std::vector<Packet> StripBoundsUpdates(const std::vector<Packet>& packets,
                                       Timestamp begin = Timestamp::Min(),
                                       Timestamp end = Timestamp::Max()) {
  std::vector<Packet> result;
  for (const auto& packet : packets) {
    Timestamp ts = packet.Timestamp();
    if (packet.IsEmpty() && ts >= begin && ts < end) {
      continue;
    }
    result.push_back(packet);
  }
  return result;
 }
 // Shows how FlowLimiterCalculator releases auxiliary input packets.
 // In this test, auxiliary input packets arrive at twice the primary rate.
 TEST_F(FlowLimiterCalculatorTest, AuxiliaryInputs) {
  // Configure the test.
  SetUpInputData();
  SetUpSimulationClock();
  CalculatorGraphConfig graph_config =
      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
        input_stream: 'input_1'
        input_stream: 'auxiliary_input_2'
        input_stream: 'auxiliary_input_3'
        node {
          calculator: 'FlowLimiterCalculator'
          options {
            [mediapipe.FlowLimiterCalculatorOptions.ext] {
              max_in_flight: 1
              max_in_queue: 0
              in_flight_timeout: 1000000  # 1s
            }
          }
          input_stream: 'input_1'
          input_stream: 'auxiliary_input_2'
          input_stream: 'auxiliary_input_3'
          input_stream: 'FINISHED:out_1'
          input_stream_info: { tag_index: 'FINISHED' back_edge: true }
          output_stream: 'input_1_sampled'
          output_stream: 'auxiliary_input_2_sampled'
          output_stream: 'auxiliary_input_3_sampled'
          output_stream: 'ALLOW:allow'
        }
        node {
          calculator: 'SleepCalculator'
          input_side_packet: 'WARMUP_TIME:warmup_time'
          input_side_packet: 'SLEEP_TIME:sleep_time'
          input_side_packet: 'CLOCK:clock'
          input_stream: 'PACKET:input_1_sampled'
          output_stream: 'PACKET:out_1'
        }
      )pb");
  std::map<std::string, Packet> side_packets = {
      // Fake processing lazy initialization time in microseconds.
      {"warmup_time", MakePacket<int64>(22000)},
      // Fake processing duration in microseconds.
      {"sleep_time", MakePacket<int64>(22000)},
      // The SimulationClock to count virtual elapsed time.
      {"clock", MakePacket<mediapipe::Clock*>(clock_)},
  };
  // Start the graph.
  MP_ASSERT_OK(graph_.Initialize(graph_config));
  MP_EXPECT_OK(graph_.ObserveOutputStream(
      "out_1",
      [this](Packet p) {
        out_1_packets_.push_back(p);
        return absl::OkStatus();
      },
      true));
  std::vector<Packet> out_2_packets, out_3_packets;
  MP_EXPECT_OK(graph_.ObserveOutputStream(
      "auxiliary_input_2_sampled",
      [&](Packet p) {
        out_2_packets.push_back(p);
        return absl::OkStatus();
      },
      true));
  MP_EXPECT_OK(graph_.ObserveOutputStream(
      "auxiliary_input_3_sampled",
      [&](Packet p) {
        out_3_packets.push_back(p);
        return absl::OkStatus();
      },
      true));
  MP_EXPECT_OK(graph_.ObserveOutputStream(
      "allow",
      [this](Packet p) {
        allow_packets_.push_back(p);
        return absl::OkStatus();
      },
      true));
  simulation_clock_->ThreadStart();
  MP_ASSERT_OK(graph_.StartRun(side_packets));
  // Add packets 1..9 to auxiliary_input_3, early.
  for (int i = 1; i < 10; ++i) {
    MP_EXPECT_OK(graph_.AddPacketToInputStream(
        "auxiliary_input_3", MakePacket<int>(i).At(Timestamp(i * 10000))));
  }
  // The total count of out_2_packets after each input packet.
  // std::vector<int> sizes_2 = {0, 0, 2, 2, 3, 3, 4, 4, 5, 5};
  std::vector<int> sizes_2 = {0, 1, 3, 4, 6, 7, 9, 10, 12, 13};
  // Add packets 2,4,6,8 to stream input_1.
  // Add packets 1..9 to auxiliary_input_2.
  for (int i = 1; i < 10; ++i) {
    if (i % 2 == 0) {
      MP_EXPECT_OK(graph_.AddPacketToInputStream(
          "input_1", MakePacket<int>(i).At(Timestamp(i * 10000))));
    }
    MP_EXPECT_OK(graph_.AddPacketToInputStream(
        "auxiliary_input_2", MakePacket<int>(i).At(Timestamp(i * 10000))));
    clock_->Sleep(absl::Microseconds(10000));
    EXPECT_EQ(out_2_packets.size(), sizes_2[i]);
  }
  // Finish the graph run.
  MP_EXPECT_OK(graph_.CloseAllPacketSources());
  clock_->Sleep(absl::Microseconds(40000));
  MP_EXPECT_OK(graph_.WaitUntilDone());
  simulation_clock_->ThreadFinish();
  // Validate the output.
  // Input packets 4 and 8 are dropped due to max_in_flight.
  std::vector<Packet> expected_output = {
      MakePacket<int>(2).At(Timestamp(20000)),
      Packet().At(Timestamp(40000)),
      MakePacket<int>(6).At(Timestamp(60000)),
      Packet().At(Timestamp(80000)),
  };
  EXPECT_THAT(out_1_packets_,
              ElementsAreArray(PacketMatchers<int>(expected_output)));
  // Packets following input packets 2 and 6, and not input packets 4 and 8.
  std::vector<Packet> expected_auxiliary_output = {
      Packet().At(Timestamp(9999)),
      MakePacket<int>(1).At(Timestamp(10000)),
      MakePacket<int>(2).At(Timestamp(20000)),
      Packet().At(Timestamp(29999)),
      MakePacket<int>(3).At(Timestamp(30000)),
      Packet().At(Timestamp(40000)),
      Packet().At(Timestamp(49999)),
      Packet().At(Timestamp(50000)),
      MakePacket<int>(6).At(Timestamp(60000)),
      Packet().At(Timestamp(69999)),
      MakePacket<int>(7).At(Timestamp(70000)),
      Packet().At(Timestamp(80000)),
      Packet().At(Timestamp(89999)),
  };
  std::vector<Packet> actual_2 =
      StripBoundsUpdates(out_2_packets, Timestamp(90000));
  EXPECT_THAT(actual_2,
              ElementsAreArray(PacketMatchers<int>(expected_auxiliary_output)));
  std::vector<Packet> expected_3 =
      StripBoundsUpdates(expected_auxiliary_output, Timestamp(39999));
  std::vector<Packet> actual_3 =
      StripBoundsUpdates(out_3_packets, Timestamp(39999));
  EXPECT_THAT(actual_3, ElementsAreArray(PacketMatchers<int>(expected_3)));
  // Validate the ALLOW stream output.
  std::vector<Packet> expected_allow = {
      MakePacket<bool>(true).At(Timestamp(20000)),
      MakePacket<bool>(false).At(Timestamp(40000)),
      MakePacket<bool>(true).At(Timestamp(60000)),
      MakePacket<bool>(false).At(Timestamp(80000)),
  };
  EXPECT_THAT(allow_packets_,
              ElementsAreArray(PacketMatchers<bool>(expected_allow)));
 }
 }  // anonymous namespace
@@ -15,6 +15,7 @@
 #include "mediapipe/calculators/core/get_vector_item_calculator.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 namespace mediapipe {
@@ -24,9 +25,17 @@ using GetLandmarkListVectorItemCalculator =
    GetVectorItemCalculator<mediapipe::LandmarkList>;
 REGISTER_CALCULATOR(GetLandmarkListVectorItemCalculator);
 using GetNormalizedLandmarkListVectorItemCalculator =
    GetVectorItemCalculator<mediapipe::NormalizedLandmarkList>;
 REGISTER_CALCULATOR(GetNormalizedLandmarkListVectorItemCalculator);
 using GetClassificationListVectorItemCalculator =
    GetVectorItemCalculator<mediapipe::ClassificationList>;
 REGISTER_CALCULATOR(GetClassificationListVectorItemCalculator);
 using GetDetectionVectorItemCalculator =
    GetVectorItemCalculator<mediapipe::Detection>;
 REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -19,6 +19,7 @@
 #include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -46,7 +47,7 @@ namespace api2 {
 //     calculator: "Get{SpecificType}VectorItemCalculator"
 //     input_stream: "VECTOR:vector"
 //     input_stream: "INDEX:index"
-//     input_stream: "ITEM:item"
+//     output_stream: "ITEM:item"
 //     options {
 //       [mediapipe.GetVectorItemCalculatorOptions.ext] {
 //         item_index: 5
@@ -58,12 +59,13 @@ template <typename T>
 class GetVectorItemCalculator : public Node {
 public:
  static constexpr Input<std::vector<T>> kIn{"VECTOR"};
-  static constexpr Input<int>::Optional kIdx{"INDEX"};
+  static constexpr Input<OneOf<int, uint64_t>>::Optional kIdx{"INDEX"};
  static constexpr Output<T> kOut{"ITEM"};
  MEDIAPIPE_NODE_CONTRACT(kIn, kIdx, kOut);
  absl::Status Open(CalculatorContext* cc) final {
    cc->SetOffset(mediapipe::TimestampDiff(0));
    auto& options = cc->Options<mediapipe::GetVectorItemCalculatorOptions>();
    RET_CHECK(kIdx(cc).IsConnected() || options.has_item_index());
    return absl::OkStatus();
@@ -80,15 +82,21 @@ class GetVectorItemCalculator : public Node {
    int idx = 0;
    if (kIdx(cc).IsConnected() && !kIdx(cc).IsEmpty()) {
-      idx = kIdx(cc).Get();
+      idx = kIdx(cc).Visit(
          [](uint64_t idx_uint64_t) { return static_cast<int>(idx_uint64_t); },
          [](int idx_int) { return idx_int; });
    } else if (options.has_item_index()) {
      idx = options.item_index();
    } else {
      return absl::OkStatus();
    }
-    RET_CHECK(idx >= 0 && idx < items.size());
+    RET_CHECK(idx >= 0);
-    kOut(cc).Send(items[idx]);
+    RET_CHECK(options.output_empty_on_oob() || idx < items.size());
    if (idx < items.size()) {
      kOut(cc).Send(items[idx]);
    }
    return absl::OkStatus();
  }
@@ -26,4 +26,7 @@ message GetVectorItemCalculatorOptions {
  // Index of vector item to get. INDEX input stream can be used instead, or to
  // override.
  optional int32 item_index = 1;
  // Set to true to output an empty packet when the index is out of bounds.
  optional bool output_empty_on_oob = 2;
 }
@@ -32,18 +32,21 @@ CalculatorRunner MakeRunnerWithStream() {
  )");
 }
-CalculatorRunner MakeRunnerWithOptions(int set_index) {
+CalculatorRunner MakeRunnerWithOptions(int set_index,
-  return CalculatorRunner(absl::StrFormat(R"(
+                                       bool output_empty_on_oob = false) {
  return CalculatorRunner(
      absl::StrFormat(R"(
    calculator: "TestGetIntVectorItemCalculator"
    input_stream: "VECTOR:vector_stream"
    output_stream: "ITEM:item_stream"
    options {
      [mediapipe.GetVectorItemCalculatorOptions.ext] {
        item_index: %d
        output_empty_on_oob: %s
      }
    }
  )",
-                                          set_index));
+                      set_index, output_empty_on_oob ? "true" : "false"));
 }
 void AddInputVector(CalculatorRunner& runner, const std::vector<int>& inputs,
@@ -140,8 +143,7 @@ TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail1) {
  absl::Status status = runner.Run();
  ASSERT_FALSE(status.ok());
-  EXPECT_THAT(status.message(),
+  EXPECT_THAT(status.message(), testing::HasSubstr("idx >= 0"));
              testing::HasSubstr("idx >= 0 && idx < items.size()"));
 }
 TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
@@ -155,7 +157,8 @@ TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
  absl::Status status = runner.Run();
  ASSERT_FALSE(status.ok());
  EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+              testing::HasSubstr(
                  "options.output_empty_on_oob() || idx < items.size()"));
 }
 TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
@@ -167,8 +170,7 @@ TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
  absl::Status status = runner.Run();
  ASSERT_FALSE(status.ok());
-  EXPECT_THAT(status.message(),
+  EXPECT_THAT(status.message(), testing::HasSubstr("idx >= 0"));
              testing::HasSubstr("idx >= 0 && idx < items.size()"));
 }
 TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
@@ -181,7 +183,21 @@ TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
  absl::Status status = runner.Run();
  ASSERT_FALSE(status.ok());
  EXPECT_THAT(status.message(),
-              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+              testing::HasSubstr(
                  "options.output_empty_on_oob() || idx < items.size()"));
 }
 TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail3) {
  const int try_index = 3;
  CalculatorRunner runner = MakeRunnerWithOptions(try_index, true);
  const std::vector<int> inputs = {1, 2, 3};
  AddInputVector(runner, inputs, 1);
  MP_ASSERT_OK(runner.Run());
  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
  EXPECT_THAT(outputs, testing::ElementsAre());
 }
 TEST(TestGetIntVectorItemCalculatorTest, IndexStreamTwoTimestamps) {
@@ -227,4 +243,15 @@ TEST(TestGetIntVectorItemCalculatorTest, IndexOptionsTwoTimestamps) {
              testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
 }
 TEST(TestGetIntVectorItemCalculatorTest, IndexUint64) {
  CalculatorRunner runner = MakeRunnerWithStream();
  const std::vector<int> inputs = {1, 2, 3};
  const uint64_t index = 1;
  AddInputVector(runner, inputs, 1);
  AddInputIndex(runner, index, 1);
  MP_ASSERT_OK(runner.Run());
  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
 }
 }  // namespace mediapipe
@@ -15,6 +15,7 @@ limitations under the License.
 #include "mediapipe/calculators/core/merge_to_vector_calculator.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image.h"
 namespace mediapipe {
@@ -23,5 +24,13 @@ namespace api2 {
 typedef MergeToVectorCalculator<mediapipe::Image> MergeImagesToVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(MergeImagesToVectorCalculator);
 typedef MergeToVectorCalculator<mediapipe::GpuBuffer>
    MergeGpuBuffersToVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(MergeGpuBuffersToVectorCalculator);
 typedef MergeToVectorCalculator<mediapipe::Detection>
    MergeDetectionsToVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(MergeDetectionsToVectorCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -42,11 +42,20 @@ class MergeToVectorCalculator : public Node {
    return absl::OkStatus();
  }
  absl::Status Open(::mediapipe::CalculatorContext* cc) {
    cc->SetOffset(::mediapipe::TimestampDiff(0));
    return absl::OkStatus();
  }
  absl::Status Process(CalculatorContext* cc) {
    const int input_num = kIn(cc).Count();
-    std::vector<T> output_vector(input_num);
+    std::vector<T> output_vector;
-    std::transform(kIn(cc).begin(), kIn(cc).end(), output_vector.begin(),
+    for (auto it = kIn(cc).begin(); it != kIn(cc).end(); it++) {
-                   [](const auto& elem) -> T { return elem.Get(); });
+      const auto& elem = *it;
      if (!elem.IsEmpty()) {
        output_vector.push_back(elem.Get());
      }
    }
    kOut(cc).Send(output_vector);
    return absl::OkStatus();
  }
@@ -41,6 +41,10 @@ class MuxCalculator : public Node {
                          StreamHandler("MuxInputStreamHandler"));
  absl::Status Process(CalculatorContext* cc) final {
    if (kSelect(cc).IsStream() && kSelect(cc).IsEmpty()) {
      return absl::OkStatus();
    }
    int select = *kSelect(cc);
    RET_CHECK(0 <= select && select < kIn(cc).Count());
    if (!kIn(cc)[select].IsEmpty()) {
@@ -398,6 +398,95 @@ TEST(MuxCalculatorTest, HandleTimestampBoundUpdates) {
  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 TEST(MuxCalculatorTest, HandlesCloseGracefully) {
  CalculatorGraphConfig config =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
          R"pb(
            input_stream: "select"
            input_stream: "value_0"
            input_stream: "value_1"
            node {
              calculator: "MuxCalculator"
              input_stream: "SELECT:select"
              input_stream: "INPUT:0:value_0"
              input_stream: "INPUT:1:value_1"
              output_stream: "OUTPUT:output"
            }
          )pb");
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(config));
  // Observe packets.
  std::vector<Packet> output_packets;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "output",
      [&output_packets](const Packet& p) -> absl::Status {
        output_packets.push_back(p);
        return absl::OkStatus();
      },
      /*observe_timestamp_bounds=*/true));
  // Start graph.
  MP_ASSERT_OK(graph.StartRun({}));
  // Add single packet wait for completion and close.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "value_0", MakePacket<int>(0).At(Timestamp(1000))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  MP_ASSERT_OK(graph.CloseAllInputStreams());
  MP_ASSERT_OK(graph.WaitUntilDone());
  EXPECT_TRUE(output_packets.empty());
 }
 TEST(MuxCalculatorTest, HandlesCloseGracefullyWithDeafultInputStreamHandler) {
  CalculatorGraphConfig config =
      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
          R"pb(
            # This is required in order for EXPECT_DEATH to work everywhere
            executor { name: "" type: "ApplicationThreadExecutor" }
            input_stream: "select"
            input_stream: "value_0"
            input_stream: "value_1"
            node {
              calculator: "MuxCalculator"
              input_stream: "SELECT:select"
              input_stream: "INPUT:0:value_0"
              input_stream: "INPUT:1:value_1"
              output_stream: "OUTPUT:output"
              input_stream_handler {
                input_stream_handler: "DefaultInputStreamHandler"
              }
            }
          )pb");
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(config));
  // Observe packets.
  std::vector<Packet> output_packets;
  MP_ASSERT_OK(graph.ObserveOutputStream(
      "output",
      [&output_packets](const Packet& p) -> absl::Status {
        output_packets.push_back(p);
        return absl::OkStatus();
      },
      /*observe_timestamp_bounds=*/true));
  // Start graph.
  MP_ASSERT_OK(graph.StartRun({}));
  // Add single packet wait for completion and close.
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "value_0", MakePacket<int>(0).At(Timestamp(1000))));
  MP_ASSERT_OK(graph.WaitUntilIdle());
  MP_ASSERT_OK(graph.CloseAllInputStreams());
  MP_ASSERT_OK(graph.WaitUntilDone());
  ASSERT_EQ(output_packets.size(), 1);
  EXPECT_TRUE(output_packets[0].IsEmpty());
 }
 }  // namespace
 }  // namespace mediapipe
@@ -76,7 +76,11 @@ constexpr char kMaxInFlightTag[] = "MAX_IN_FLIGHT";
 //   }
 //   output_stream: "gated_frames"
 // }
-class RealTimeFlowLimiterCalculator : public CalculatorBase {
+//
 // Please use FlowLimiterCalculator, which replaces this calculator and
 // defines a few additional configuration options.
 class ABSL_DEPRECATED("Use FlowLimiterCalculator instead.")
    RealTimeFlowLimiterCalculator : public CalculatorBase {
 public:
  static absl::Status GetContract(CalculatorContract* cc) {
    int num_data_streams = cc->Inputs().NumEntries("");
@@ -66,12 +66,16 @@ class SequenceShiftCalculator : public Node {
  // The number of packets or timestamps we need to store to output packet[i] at
  // the timestamp of packet[i + packet_offset]; equal to abs(packet_offset).
  int cache_size_;
  bool emit_empty_packets_before_first_packet_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(SequenceShiftCalculator);
 absl::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
  packet_offset_ = kOffset(cc).GetOr(
      cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset());
  emit_empty_packets_before_first_packet_ =
      cc->Options<mediapipe::SequenceShiftCalculatorOptions>()
          .emit_empty_packets_before_first_packet();
  cache_size_ = abs(packet_offset_);
  // An offset of zero is a no-op, but someone might still request it.
  if (packet_offset_ == 0) {
@@ -96,6 +100,8 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
    // Ready to output oldest packet with current timestamp.
    kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
    packet_cache_.pop_front();
  } else if (emit_empty_packets_before_first_packet_) {
    LOG(FATAL) << "Not supported yet";
  }
  // Store current packet for later output.
  packet_cache_.push_back(kIn(cc).packet());
@@ -23,4 +23,8 @@ message SequenceShiftCalculatorOptions {
    optional SequenceShiftCalculatorOptions ext = 107633927;
  }
  optional int32 packet_offset = 1 [default = -1];
  // Emits empty packets before the first delayed packet is emitted. Takes
  // effect only when packet offset is set to positive.
  optional bool emit_empty_packets_before_first_packet = 2 [default = false];
 }
@@ -0,0 +1,33 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include "mediapipe/calculators/core/vector_indices_calculator.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 namespace mediapipe {
 namespace api2 {
 using IntVectorIndicesCalculator = VectorIndicesCalculator<int>;
 REGISTER_CALCULATOR(IntVectorIndicesCalculator);
 using Uint64tVectorIndicesCalculator = VectorIndicesCalculator<uint64_t>;
 REGISTER_CALCULATOR(Uint64tVectorIndicesCalculator);
 using NormalizedLandmarkListVectorIndicesCalculator =
    VectorIndicesCalculator<mediapipe::NormalizedLandmarkList>;
 REGISTER_CALCULATOR(NormalizedLandmarkListVectorIndicesCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,65 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #ifndef MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_
 #include <optional>
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 namespace mediapipe {
 namespace api2 {
 // Calculator that takes a vector and constructs an index range vector based on
 // the size of the input vector.
 //
 // Inputs:
 //   VECTOR - std::vector<T>
 //     Vector whose range of indices to return.
 //
 // Outputs:
 //   INDICES - std::vector<int>
 //     Indices vector of the input vector.
 //
 // Example config:
 //  node {
 //    calculator: "{SpecificType}VectorIndicesCalculator"
 //    input_stream: "VECTOR:vector"
 //    output_stream: "INDICES:indices"
 //  }
 //
 template <typename T>
 class VectorIndicesCalculator : public Node {
 public:
  static constexpr Input<std::vector<T>> kVector{"VECTOR"};
  static constexpr Output<std::vector<int>> kRange{"INDICES"};
  MEDIAPIPE_NODE_CONTRACT(kVector, kRange);
  absl::Status Process(CalculatorContext* cc) final {
    // Get the size of the input vector.
    const int vector_size = kVector(cc).Get().size();
    std::vector<int> out_idxs(vector_size);
    std::iota(out_idxs.begin(), out_idxs.end(), 0);
    kRange(cc).Send(out_idxs);
    return absl::OkStatus();
  }
 };
 }  // namespace api2
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_CORE_VECTOR_INDICES_CALCULATOR_H_
@@ -0,0 +1,87 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include "mediapipe/calculators/core/vector_indices_calculator.h"
 #include <memory>
 #include <string>
 #include <vector>
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/status_matchers.h"
 namespace mediapipe {
 namespace {
 using ::testing::TestParamInfo;
 using ::testing::TestWithParam;
 using ::testing::Values;
 template <typename T>
 void AddInputVector(CalculatorRunner& runner, const std::vector<T>& inputs,
                    int timestamp) {
  runner.MutableInputs()->Tag("VECTOR").packets.push_back(
      MakePacket<std::vector<T>>(inputs).At(Timestamp(timestamp)));
 }
 template <typename T>
 struct TestParams {
  const std::string test_name;
  const std::vector<T> inputs;
  const int timestamp;
  const std::vector<int> expected_indices;
 };
 class IntVectorIndicesCalculatorTest
    : public testing::TestWithParam<TestParams<int>> {};
 TEST_P(IntVectorIndicesCalculatorTest, Succeeds) {
  CalculatorRunner runner = CalculatorRunner(R"(
    calculator: "IntVectorIndicesCalculator"
    input_stream: "VECTOR:vector_stream"
    output_stream: "INDICES:indices_stream"
  )");
  const std::vector<int>& inputs = GetParam().inputs;
  std::vector<int> expected_indices(inputs.size());
  AddInputVector(runner, inputs, GetParam().timestamp);
  MP_ASSERT_OK(runner.Run());
  const std::vector<Packet>& outputs = runner.Outputs().Tag("INDICES").packets;
  EXPECT_EQ(1, outputs.size());
  EXPECT_THAT(outputs[0].Get<std::vector<int>>(),
              testing::ElementsAreArray(GetParam().expected_indices));
 }
 INSTANTIATE_TEST_SUITE_P(
    IntVectorIndicesCalculatorTest, IntVectorIndicesCalculatorTest,
    Values(TestParams<int>{
               /* test_name= */ "IntVectorIndices",
               /* inputs= */ {1, 2, 3},
               /* timestamp= */ 1,
               /* expected_indices= */ {0, 1, 2},
           },
           TestParams<int>{
               /* test_name= */ "EmptyVector",
               /* inputs= */ {},
               /* timestamp= */ 1,
               /* expected_indices= */ {},
           }),
    [](const TestParamInfo<IntVectorIndicesCalculatorTest::ParamType>& info) {
      return info.param.test_name;
    });
 }  // namespace
 }  // namespace mediapipe
@@ -16,12 +16,11 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 licenses(["notice"])
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 mediapipe_proto_library(
    name = "opencv_image_encoder_calculator_proto",
    srcs = ["opencv_image_encoder_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -31,7 +30,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "scale_image_calculator_proto",
    srcs = ["scale_image_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -42,7 +40,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "set_alpha_calculator_proto",
    srcs = ["set_alpha_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -52,7 +49,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "image_cropping_calculator_proto",
    srcs = ["image_cropping_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -62,7 +58,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "bilateral_filter_calculator_proto",
    srcs = ["bilateral_filter_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -72,7 +67,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "recolor_calculator_proto",
    srcs = ["recolor_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -83,7 +77,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "segmentation_smoothing_calculator_proto",
    srcs = ["segmentation_smoothing_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -93,7 +86,6 @@ mediapipe_proto_library(
 cc_library(
    name = "color_convert_calculator",
    srcs = ["color_convert_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:timestamp",
@@ -112,7 +104,6 @@ cc_library(
 cc_library(
    name = "opencv_encoded_image_to_image_frame_calculator",
    srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":opencv_encoded_image_to_image_frame_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -127,7 +118,6 @@ cc_library(
 cc_library(
    name = "opencv_image_encoder_calculator",
    srcs = ["opencv_image_encoder_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":opencv_image_encoder_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -142,7 +132,6 @@ cc_library(
 cc_library(
    name = "opencv_put_text_calculator",
    srcs = ["opencv_put_text_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame_opencv",
@@ -156,11 +145,10 @@ cc_library(
 cc_library(
    name = "set_alpha_calculator",
    srcs = ["set_alpha_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":set_alpha_calculator_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "//mediapipe/framework/formats:image_format_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
@@ -183,11 +171,10 @@ cc_library(
 cc_library(
    name = "bilateral_filter_calculator",
    srcs = ["bilateral_filter_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":bilateral_filter_calculator_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "//mediapipe/framework/formats:image_format_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "@com_google_absl//absl/strings",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
@@ -209,11 +196,16 @@ cc_library(
    alwayslink = 1,
 )
 mediapipe_proto_library(
    name = "rotation_mode_proto",
    srcs = ["rotation_mode.proto"],
 )
 mediapipe_proto_library(
    name = "image_transformation_calculator_proto",
    srcs = ["image_transformation_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        ":rotation_mode_proto",
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
        "//mediapipe/gpu:scale_mode_proto",
@@ -236,8 +228,8 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":rotation_mode_cc_proto",
        ":image_transformation_calculator_cc_proto",
        "//mediapipe/framework:packet",
        "//mediapipe/framework:timestamp",
@@ -279,13 +271,12 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":image_cropping_calculator_cc_proto",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework/port:opencv_core",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:ret_check",
@@ -322,7 +313,6 @@ cc_test(
 cc_library(
    name = "luminance_calculator",
    srcs = ["luminance_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
@@ -336,7 +326,6 @@ cc_library(
 cc_library(
    name = "sobel_edges_calculator",
    srcs = ["sobel_edges_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
@@ -350,15 +339,14 @@ cc_library(
 cc_library(
    name = "recolor_calculator",
    srcs = ["recolor_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":recolor_calculator_cc_proto",
        "//mediapipe/util:color_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/port:status",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/util:color_cc_proto",
        "//mediapipe/framework/port:opencv_core",
        "//mediapipe/framework/port:opencv_imgproc",
    ] + select({
@@ -377,9 +365,6 @@ cc_library(
    name = "scale_image_utils",
    srcs = ["scale_image_utils.cc"],
    hdrs = ["scale_image_utils.h"],
    visibility = [
        "//mediapipe:__subpackages__",
    ],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/port:logging",
@@ -392,12 +377,9 @@ cc_library(
 cc_library(
    name = "scale_image_calculator",
    srcs = ["scale_image_calculator.cc"],
    visibility = [
        "//visibility:public",
    ],
    deps = [
        ":scale_image_calculator_cc_proto",
        ":scale_image_utils",
        "//mediapipe/calculators/image:scale_image_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_format_cc_proto",
        "//mediapipe/framework/formats:image_frame",
@@ -421,7 +403,6 @@ cc_library(
 mediapipe_proto_library(
    name = "image_clone_calculator_proto",
    srcs = ["image_clone_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -431,7 +412,6 @@ mediapipe_proto_library(
 cc_library(
    name = "image_clone_calculator",
    srcs = ["image_clone_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":image_clone_calculator_cc_proto",
        "//mediapipe/framework/api2:node",
@@ -451,7 +431,6 @@ cc_library(
 cc_library(
    name = "image_properties_calculator",
    srcs = ["image_properties_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework:calculator_framework",
@@ -516,7 +495,6 @@ cc_test(
 mediapipe_proto_library(
    name = "mask_overlay_calculator_proto",
    srcs = ["mask_overlay_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -526,7 +504,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "opencv_encoded_image_to_image_frame_calculator_proto",
    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -536,7 +513,6 @@ mediapipe_proto_library(
 mediapipe_proto_library(
    name = "feature_detector_calculator_proto",
    srcs = ["feature_detector_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -546,7 +522,6 @@ mediapipe_proto_library(
 cc_library(
    name = "mask_overlay_calculator",
    srcs = ["mask_overlay_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":mask_overlay_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -562,7 +537,6 @@ cc_library(
 cc_library(
    name = "feature_detector_calculator",
    srcs = ["feature_detector_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":feature_detector_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -589,7 +563,6 @@ cc_library(
 cc_library(
    name = "image_file_properties_calculator",
    srcs = ["image_file_properties_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_file_properties_cc_proto",
@@ -619,11 +592,10 @@ cc_test(
 cc_library(
    name = "segmentation_smoothing_calculator",
    srcs = ["segmentation_smoothing_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        ":segmentation_smoothing_calculator_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "//mediapipe/framework/formats:image_format_cc_proto",
        "//mediapipe/framework:calculator_options_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame",
        "//mediapipe/framework/formats:image",
@@ -716,7 +688,6 @@ cc_library(
 mediapipe_proto_library(
    name = "warp_affine_calculator_proto",
    srcs = ["warp_affine_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -728,7 +699,6 @@ cc_library(
    name = "warp_affine_calculator",
    srcs = ["warp_affine_calculator.cc"],
    hdrs = ["warp_affine_calculator.h"],
    visibility = ["//visibility:public"],
    deps = [
        ":affine_transformation",
        ":warp_affine_calculator_cc_proto",
@@ -777,8 +747,8 @@ cc_test(
    tags = ["desktop_only_test"],
    deps = [
        ":affine_transformation",
        ":image_transformation_calculator",
        ":warp_affine_calculator",
        "//mediapipe/calculators/image:image_transformation_calculator",
        "//mediapipe/calculators/tensor:image_to_tensor_converter",
        "//mediapipe/calculators/tensor:image_to_tensor_utils",
        "//mediapipe/calculators/util:from_image_calculator",
@@ -809,7 +779,6 @@ cc_test(
 cc_library(
    name = "yuv_to_image_calculator",
    srcs = ["yuv_to_image_calculator.cc"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
@@ -92,8 +92,8 @@ class GlTextureWarpAffineRunner
      constexpr GLchar kVertShader[] = R"(
            in vec4 position;
-            in mediump vec4 texture_coordinate;
+            in highp vec4 texture_coordinate;
-            out mediump vec2 sample_coordinate;
+            out highp vec2 sample_coordinate;
            uniform mat4 transform_matrix;
            void main() {
@@ -104,7 +104,7 @@ class GlTextureWarpAffineRunner
          )";
      constexpr GLchar kFragShader[] = R"(
-            DEFAULT_PRECISION(mediump, float)
+            DEFAULT_PRECISION(highp, float)
            in vec2 sample_coordinate;
            uniform sampler2D input_texture;
@@ -38,6 +38,7 @@ void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
 constexpr char kRgbaInTag[] = "RGBA_IN";
 constexpr char kRgbInTag[] = "RGB_IN";
 constexpr char kBgrInTag[] = "BGR_IN";
 constexpr char kBgraInTag[] = "BGRA_IN";
 constexpr char kGrayInTag[] = "GRAY_IN";
 constexpr char kRgbaOutTag[] = "RGBA_OUT";
@@ -57,6 +58,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 //   RGB  -> RGBA
 //   RGBA -> BGRA
 //   BGRA -> RGBA
 //   BGR  -> RGB
 //
 // This calculator only supports a single input stream and output stream at a
 // time. If more than one input stream or output stream is present, the
@@ -69,6 +71,7 @@ constexpr char kGrayOutTag[] = "GRAY_OUT";
 //   RGB_IN:        The input video stream (ImageFrame, SRGB).
 //   BGRA_IN:       The input video stream (ImageFrame, SBGRA).
 //   GRAY_IN:       The input video stream (ImageFrame, GRAY8).
 //   BGR_IN:        The input video stream (ImageFrame, SBGR).
 //
 // Output streams:
 //   RGBA_OUT:      The output video stream (ImageFrame, SRGBA).
@@ -122,6 +125,10 @@ absl::Status ColorConvertCalculator::GetContract(CalculatorContract* cc) {
    cc->Inputs().Tag(kBgraInTag).Set<ImageFrame>();
  }
  if (cc->Inputs().HasTag(kBgrInTag)) {
    cc->Inputs().Tag(kBgrInTag).Set<ImageFrame>();
  }
  if (cc->Outputs().HasTag(kRgbOutTag)) {
    cc->Outputs().Tag(kRgbOutTag).Set<ImageFrame>();
  }
@@ -194,6 +201,11 @@ absl::Status ColorConvertCalculator::Process(CalculatorContext* cc) {
    return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA,
                            cv::COLOR_RGBA2BGRA, cc);
  }
  // BGR -> RGB
  if (cc->Inputs().HasTag(kBgrInTag) && cc->Outputs().HasTag(kRgbOutTag)) {
    return ConvertAndOutput(kBgrInTag, kRgbOutTag, ImageFormat::SRGB,
                            cv::COLOR_BGR2RGB, cc);
  }
  return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
         << "Unsupported image format conversion.";
@@ -37,7 +37,8 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 namespace mediapipe {
 namespace {
-
+using ::mediapipe::NormalizedRect;
 using ::mediapipe::Rect;
 #if !MEDIAPIPE_DISABLE_GPU
 #endif  // !MEDIAPIPE_DISABLE_GPU
@@ -195,11 +195,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
  auto cc = absl::make_unique<CalculatorContext>(
      calculator_state.get(), inputTags, tool::CreateTagMap({}).value());
  auto& inputs = cc->Inputs();
-  mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
+  Rect rect = ParseTextProtoOrDie<Rect>(
      R"pb(
        width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
      )pb");
-  inputs.Tag(kRectTag).Value() = MakePacket<mediapipe::Rect>(rect);
+  inputs.Tag(kRectTag).Value() = MakePacket<Rect>(rect);
  RectSpec expectRect = {
      .width = 1,
      .height = 1,
@@ -13,6 +13,7 @@
 // limitations under the License.
 #include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
 #include "mediapipe/calculators/image/rotation_mode.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -16,20 +16,10 @@ syntax = "proto2";
 package mediapipe;
 import "mediapipe/calculators/image/rotation_mode.proto";
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/gpu/scale_mode.proto";
 // Counterclockwise rotation.
 message RotationMode {
  enum Mode {
    UNKNOWN = 0;
    ROTATION_0 = 1;
    ROTATION_90 = 2;
    ROTATION_180 = 3;
    ROTATION_270 = 4;
  }
 }
 message ImageTransformationCalculatorOptions {
  extend CalculatorOptions {
    optional ImageTransformationCalculatorOptions ext = 251952830;
@@ -0,0 +1,31 @@
 // Copyright 2019 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 syntax = "proto2";
 package mediapipe;
 option java_package = "com.google.mediapipe.calculator.proto";
 option java_outer_classname = "RotationModeProto";
 // Counterclockwise rotation.
 message RotationMode {
  enum Mode {
    UNKNOWN = 0;
    ROTATION_0 = 1;
    ROTATION_90 = 2;
    ROTATION_180 = 3;
    ROTATION_270 = 4;
  }
 }
@@ -142,6 +142,9 @@ absl::Status FindOutputDimensions(int input_width,             //
                                      static_cast<double>(input_height));
    try_width = (try_width / 2) * 2;
    try_height = (try_height / 2) * 2;
    // The output width/height should be greater than 0.
    try_width = std::max(try_width, 1);
    try_height = std::max(try_height, 1);
    if (target_height <= 0 || try_height <= target_height) {
      // The resulting height based on the target width and aspect ratio
@@ -160,6 +163,9 @@ absl::Status FindOutputDimensions(int input_width,             //
                                     static_cast<double>(input_width));
    try_width = (try_width / 2) * 2;
    try_height = (try_height / 2) * 2;
    // The output width/height should be greater than 0.
    try_width = std::max(try_width, 1);
    try_height = std::max(try_height, 1);
    if (target_width <= 0 || try_width <= target_width) {
      // The resulting width based on the target width and aspect ratio
@@ -124,6 +124,16 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
                                    &output_width, &output_height));
  EXPECT_EQ(151, output_width);
  EXPECT_EQ(101, output_height);
  // Scale to height 1.
  MP_ASSERT_OK(FindOutputDimensions(10000, 10, 100, 0, 0, true, 2,
                                    &output_width, &output_height));
  EXPECT_EQ(100, output_width);
  EXPECT_EQ(1, output_height);
  // Scale to width 1.
  MP_ASSERT_OK(FindOutputDimensions(10, 10000, 0, 100, 0, true, 2,
                                    &output_width, &output_height));
  EXPECT_EQ(1, output_width);
  EXPECT_EQ(100, output_height);
 }
 // Tests scaling without keeping the aspect ratio fixed.
@@ -46,6 +46,40 @@ constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
 constexpr int kNumChannelsRGBA = 4;
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 // Combines an RGB cv::Mat and a single-channel alpha cv::Mat of the same
 // dimensions into an RGBA cv::Mat. Alpha may be read as uint8 or as another
 // numeric type; in the latter case, it is upscaled to values between 0 and 255
 // from an assumed input range of [0, 1). RGB and RGBA Mat's must be uchar.
 template <typename AlphaType>
 absl::Status MergeRGBA8Image(const cv::Mat input_mat, const cv::Mat& alpha_mat,
                             cv::Mat& output_mat) {
  RET_CHECK_EQ(input_mat.rows, alpha_mat.rows);
  RET_CHECK_EQ(input_mat.cols, alpha_mat.cols);
  RET_CHECK_EQ(input_mat.rows, output_mat.rows);
  RET_CHECK_EQ(input_mat.cols, output_mat.cols);
  for (int i = 0; i < output_mat.rows; ++i) {
    const uchar* in_ptr = input_mat.ptr<uchar>(i);
    const AlphaType* alpha_ptr = alpha_mat.ptr<AlphaType>(i);
    uchar* out_ptr = output_mat.ptr<uchar>(i);
    for (int j = 0; j < output_mat.cols; ++j) {
      const int out_idx = j * kNumChannelsRGBA;
      const int in_idx = j * input_mat.channels();
      const int alpha_idx = j * alpha_mat.channels();
      out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
      out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
      out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
      if constexpr (std::is_same<AlphaType, uchar>::value) {
        out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0];
      } else {
        const AlphaType alpha = alpha_ptr[alpha_idx + 0];
        out_ptr[out_idx + 3] = static_cast<uchar>(round(alpha * 255.0f));
      }
    }
  }
  return absl::OkStatus();
 }
 }  // namespace
 // A calculator for setting the alpha channel of an RGBA image.
@@ -250,28 +284,22 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
  const bool has_alpha_mask = cc->Inputs().HasTag(kInputAlphaTag) &&
                              !cc->Inputs().Tag(kInputAlphaTag).IsEmpty();
-  const bool use_alpa_mask = alpha_value_ < 0 && has_alpha_mask;
+  const bool use_alpha_mask = alpha_value_ < 0 && has_alpha_mask;
  // Setup alpha image and Update image in CPU.
-  if (use_alpa_mask) {
+  if (use_alpha_mask) {
    const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get<ImageFrame>();
    cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask);
    RET_CHECK_EQ(input_mat.rows, alpha_mat.rows);
    RET_CHECK_EQ(input_mat.cols, alpha_mat.cols);
-    for (int i = 0; i < output_mat.rows; ++i) {
+    const bool alpha_is_float = alpha_mat.type() == CV_32FC1;
-      const uchar* in_ptr = input_mat.ptr<uchar>(i);
+    RET_CHECK(alpha_is_float || alpha_mat.type() == CV_8UC1);
-      uchar* alpha_ptr = alpha_mat.ptr<uchar>(i);
+
-      uchar* out_ptr = output_mat.ptr<uchar>(i);
+    if (alpha_is_float) {
-      for (int j = 0; j < output_mat.cols; ++j) {
+      MP_RETURN_IF_ERROR(
-        const int out_idx = j * kNumChannelsRGBA;
+          MergeRGBA8Image<float>(input_mat, alpha_mat, output_mat));
-        const int in_idx = j * input_mat.channels();
+    } else {
-        const int alpha_idx = j * alpha_mat.channels();
+      MP_RETURN_IF_ERROR(
-        out_ptr[out_idx + 0] = in_ptr[in_idx + 0];
+          MergeRGBA8Image<uchar>(input_mat, alpha_mat, output_mat));
        out_ptr[out_idx + 1] = in_ptr[in_idx + 1];
        out_ptr[out_idx + 2] = in_ptr[in_idx + 2];
        out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0];  // channel 0 of mask
      }
    }
  } else {
    const uchar alpha_value = std::min(std::max(0.0f, alpha_value_), 255.0f);
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 licenses(["notice"])
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
 licenses(["notice"])
 package(default_visibility = ["//visibility:private"])
 proto_library(
    name = "callback_packet_calculator_proto",
    srcs = ["callback_packet_calculator.proto"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
    deps = ["//mediapipe/framework:calculator_proto"],
 )
@@ -29,14 +29,14 @@ mediapipe_cc_proto_library(
    name = "callback_packet_calculator_cc_proto",
    srcs = ["callback_packet_calculator.proto"],
    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
    deps = [":callback_packet_calculator_proto"],
 )
 cc_library(
    name = "callback_packet_calculator",
    srcs = ["callback_packet_calculator.cc"],
-    visibility = ["//visibility:public"],
+    visibility = ["//mediapipe/framework:__subpackages__"],
    deps = [
        ":callback_packet_calculator_cc_proto",
        "//mediapipe/framework:calculator_base",
@@ -24,12 +24,13 @@ load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
 licenses(["notice"])
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 exports_files(
    glob(["testdata/image_to_tensor/*"]),
    visibility = [
        "//mediapipe/calculators/image:__subpackages__",
        "//mediapipe/util:__subpackages__",
    ],
 )
@@ -43,9 +44,6 @@ selects.config_setting_group(
 mediapipe_proto_library(
    name = "audio_to_tensor_calculator_proto",
    srcs = ["audio_to_tensor_calculator.proto"],
    visibility = [
        "//mediapipe/framework:mediapipe_internal",
    ],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -55,9 +53,6 @@ mediapipe_proto_library(
 cc_library(
    name = "audio_to_tensor_calculator",
    srcs = ["audio_to_tensor_calculator.cc"],
    visibility = [
        "//mediapipe/framework:mediapipe_internal",
    ],
    deps = [
        ":audio_to_tensor_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -67,13 +62,16 @@ cc_library(
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/formats:time_series_header_cc_proto",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/util:time_series_util",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings:str_format",
        "@com_google_audio_tools//audio/dsp:resampler_q",
        "@com_google_audio_tools//audio/dsp:window_functions",
        "@org_tensorflow//tensorflow/lite/c:common",
        "@pffft",
    ],
    alwayslink = 1,
 )
@@ -83,6 +81,7 @@ cc_test(
    srcs = ["audio_to_tensor_calculator_test.cc"],
    deps = [
        ":audio_to_tensor_calculator",
        ":audio_to_tensor_calculator_cc_proto",
        "//mediapipe/framework:calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:timestamp",
@@ -97,10 +96,266 @@ cc_test(
    ],
 )
 mediapipe_proto_library(
    name = "tensors_to_audio_calculator_proto",
    srcs = ["tensors_to_audio_calculator.proto"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
    ],
 )
 cc_library(
    name = "tensors_to_audio_calculator",
    srcs = ["tensors_to_audio_calculator.cc"],
    deps = [
        ":tensors_to_audio_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
        "@com_google_absl//absl/algorithm:container",
        "@com_google_absl//absl/status",
        "@com_google_audio_tools//audio/dsp:window_functions",
        "@pffft",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "tensors_to_audio_calculator_test",
    srcs = ["tensors_to_audio_calculator_test.cc"],
    deps = [
        ":audio_to_tensor_calculator",
        ":audio_to_tensor_calculator_cc_proto",
        ":tensors_to_audio_calculator",
        ":tensors_to_audio_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings",
        "@com_google_audio_tools//audio/dsp:window_functions",
    ],
 )
 mediapipe_proto_library(
    name = "feedback_tensors_calculator_proto",
    srcs = ["feedback_tensors_calculator.proto"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
    ],
 )
 cc_library(
    name = "feedback_tensors_calculator",
    srcs = ["feedback_tensors_calculator.cc"],
    deps = [
        ":feedback_tensors_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/formats:tensor",
        "@com_google_absl//absl/status",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "feedback_tensors_calculator_test",
    srcs = ["feedback_tensors_calculator_test.cc"],
    deps = [
        ":feedback_tensors_calculator",
        ":feedback_tensors_calculator_cc_proto",
        "//mediapipe/framework:calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:timestamp",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "@org_tensorflow//tensorflow/lite/c:common",
    ],
 )
 mediapipe_proto_library(
    name = "bert_preprocessor_calculator_proto",
    srcs = ["bert_preprocessor_calculator.proto"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
    ],
 )
 cc_library(
    name = "bert_preprocessor_calculator",
    srcs = ["bert_preprocessor_calculator.cc"],
    deps = [
        ":bert_preprocessor_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/tasks/cc/core:utils",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "//mediapipe/tasks/cc/text/tokenizers:tokenizer",
        "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
        "//mediapipe/tasks/metadata:metadata_schema_cc",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "bert_preprocessor_calculator_test",
    srcs = ["bert_preprocessor_calculator_test.cc"],
    data = ["//mediapipe/tasks/testdata/text:bert_text_classifier_models"],
    linkopts = ["-ldl"],
    deps = [
        ":bert_preprocessor_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/tasks/cc/core:utils",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_sentencepiece//src:sentencepiece_processor",
    ],
 )
 mediapipe_proto_library(
    name = "regex_preprocessor_calculator_proto",
    srcs = ["regex_preprocessor_calculator.proto"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
    ],
 )
 cc_library(
    name = "regex_preprocessor_calculator",
    srcs = ["regex_preprocessor_calculator.cc"],
    deps = [
        ":regex_preprocessor_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "//mediapipe/tasks/cc/text/tokenizers:regex_tokenizer",
        "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
        "//mediapipe/tasks/metadata:metadata_schema_cc",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "regex_preprocessor_calculator_test",
    srcs = ["regex_preprocessor_calculator_test.cc"],
    data = ["//mediapipe/tasks/testdata/text:text_classifier_models"],
    linkopts = ["-ldl"],
    deps = [
        ":regex_preprocessor_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/tool:sink",
        "//mediapipe/tasks/cc/core:utils",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
        "@com_google_sentencepiece//src:sentencepiece_processor",
    ],
 )
 cc_library(
    name = "text_to_tensor_calculator",
    srcs = ["text_to_tensor_calculator.cc"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:tensor",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "text_to_tensor_calculator_test",
    srcs = ["text_to_tensor_calculator_test.cc"],
    deps = [
        ":text_to_tensor_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_graph",
        "//mediapipe/framework:packet",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/tool:options_map",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
    ],
 )
 cc_library(
    name = "universal_sentence_encoder_preprocessor_calculator",
    srcs = ["universal_sentence_encoder_preprocessor_calculator.cc"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/tasks/cc/core:utils",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings",
    ],
    alwayslink = 1,
 )
 cc_test(
    name = "universal_sentence_encoder_preprocessor_calculator_test",
    srcs = ["universal_sentence_encoder_preprocessor_calculator_test.cc"],
    data = ["//mediapipe/tasks/testdata/text:universal_sentence_encoder_qa"],
    deps = [
        ":universal_sentence_encoder_preprocessor_calculator",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:packet",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
        "//mediapipe/framework/tool:options_map",
        "//mediapipe/tasks/cc/core:utils",
        "//mediapipe/tasks/cc/metadata:metadata_extractor",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
    ],
 )
 mediapipe_proto_library(
    name = "inference_calculator_proto",
    srcs = ["inference_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -119,16 +374,8 @@ cc_library(
    name = "inference_calculator_interface",
    srcs = ["inference_calculator.cc"],
    hdrs = ["inference_calculator.h"],
    copts = select({
        # TODO: fix tensor.h not to require this, if possible
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_cc_proto",
        ":inference_calculator_options_lib",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/api2:node",
@@ -151,7 +398,6 @@ cc_library(
    name = "inference_calculator_gl",
    srcs = ["inference_calculator_gl.cc"],
    tags = ["nomac"],  # config problem with cpuinfo via TF
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_cc_proto",
        ":inference_calculator_interface",
@@ -159,6 +405,7 @@ cc_library(
        "//mediapipe/gpu:gl_calculator_helper",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/strings:str_format",
        "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
    ],
    alwayslink = 1,
@@ -168,7 +415,6 @@ cc_library(
    name = "inference_calculator_gl_advanced",
    srcs = ["inference_calculator_gl_advanced.cc"],
    tags = ["nomac"],
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_interface",
        "@com_google_absl//absl/memory",
@@ -199,15 +445,16 @@ cc_library(
        "-framework MetalKit",
    ],
    tags = ["ios"],
    visibility = ["//visibility:public"],
    deps = [
        "inference_calculator_interface",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/gpu:MPPMetalHelper",
        "//mediapipe/gpu:MPPMetalUtil",
        "//mediapipe/gpu:gpu_buffer",
        "//mediapipe/objc:mediapipe_framework_ios",
        "//mediapipe/util/tflite:config",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/strings:str_format",
        "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
        "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
        "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
@@ -219,16 +466,8 @@ cc_library(
 cc_library(
    name = "inference_runner",
    hdrs = ["inference_runner.h"],
    copts = select({
        # TODO: fix tensor.h not to require this, if possible
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework/formats:tensor",
        "@com_google_absl//absl/status:statusor",
    ],
@@ -238,17 +477,9 @@ cc_library(
    name = "inference_interpreter_delegate_runner",
    srcs = ["inference_interpreter_delegate_runner.cc"],
    hdrs = ["inference_interpreter_delegate_runner.h"],
    copts = select({
        # TODO: fix tensor.h not to require this, if possible
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":inference_runner",
        "//mediapipe/framework:mediapipe_profiling",
        "//mediapipe/framework/api2:packet",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
@@ -256,6 +487,8 @@ cc_library(
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/status:statusor",
        "@org_tensorflow//tensorflow/lite:framework_stable",
        "@org_tensorflow//tensorflow/lite:string_util",
        "@org_tensorflow//tensorflow/lite/c:c_api_types",
        "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
    ],
 )
@@ -265,15 +498,6 @@ cc_library(
    srcs = [
        "inference_calculator_cpu.cc",
    ],
    copts = select({
        # TODO: fix tensor.h not to require this, if possible
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_interface",
        ":inference_calculator_utils",
@@ -312,15 +536,6 @@ cc_library(
    srcs = [
        "inference_calculator_xnnpack.cc",
    ],
    copts = select({
        # TODO: fix tensor.h not to require this, if possible
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_interface",
        ":inference_calculator_utils",
@@ -336,7 +551,6 @@ cc_library(
 cc_library(
    name = "inference_calculator_gl_if_compute_shader_available",
    visibility = ["//visibility:public"],
    deps = selects.with_or({
        ":compute_shader_unavailable": [],
        "//conditions:default": [
@@ -346,9 +560,12 @@ cc_library(
    }),
 )
 # This target provides the InferenceCalculator and a default set of implementations tailored for the
 # current build platforms. More implementations can be added as separate dependencies to a client;
 # for clients that want a narrower set of implementations than the default should see the comment on
 # inference_calculator_interface.
 cc_library(
    name = "inference_calculator",
    visibility = ["//visibility:public"],
    deps = [
        ":inference_calculator_interface",
        ":inference_calculator_cpu",
@@ -362,7 +579,6 @@ cc_library(
 mediapipe_proto_library(
    name = "tensor_converter_calculator_proto",
    srcs = ["tensor_converter_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -387,7 +603,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensor_converter_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -406,6 +621,7 @@ cc_library(
 cc_library(
    name = "tensor_converter_calculator_gpu_deps",
    visibility = ["//visibility:private"],
    deps = select({
        "//mediapipe:android": [
            "//mediapipe/gpu:gl_calculator_helper",
@@ -450,7 +666,6 @@ cc_test(
 mediapipe_proto_library(
    name = "tensors_to_detections_calculator_proto",
    srcs = ["tensors_to_detections_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -475,19 +690,18 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensors_to_detections_calculator_cc_proto",
        "//mediapipe/framework/formats:detection_cc_proto",
        "@com_google_absl//absl/strings:str_format",
        "@com_google_absl//absl/types:span",
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:port",
        "//mediapipe/framework/deps:file_path",
        "//mediapipe/framework/formats:location",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
        "//mediapipe/framework/port:ret_check",
    ] + selects.with_or({
        ":compute_shader_unavailable": [],
@@ -498,6 +712,7 @@ cc_library(
 cc_library(
    name = "tensors_to_detections_calculator_gpu_deps",
    visibility = ["//visibility:private"],
    deps = select({
        "//mediapipe:ios": [
            "//mediapipe/gpu:MPPMetalUtil",
@@ -513,7 +728,6 @@ cc_library(
 mediapipe_proto_library(
    name = "tensors_to_landmarks_calculator_proto",
    srcs = ["tensors_to_landmarks_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -530,7 +744,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensors_to_landmarks_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -545,7 +758,6 @@ cc_library(
 mediapipe_proto_library(
    name = "landmarks_to_tensor_calculator_proto",
    srcs = ["landmarks_to_tensor_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -563,7 +775,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":landmarks_to_tensor_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -596,7 +807,6 @@ cc_test(
 mediapipe_proto_library(
    name = "tensors_to_floats_calculator_proto",
    srcs = ["tensors_to_floats_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -613,7 +823,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensors_to_floats_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
@@ -651,7 +860,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensors_to_classification_calculator_cc_proto",
        "@com_google_absl//absl/container:node_hash_map",
@@ -682,7 +890,6 @@ cc_library(
 mediapipe_proto_library(
    name = "tensors_to_classification_calculator_proto",
    srcs = ["tensors_to_classification_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -720,7 +927,6 @@ cc_library(
        "//conditions:default": [],
    }),
    features = ["-layering_check"],  # allow depending on image_to_tensor_calculator_gpu_deps
    visibility = ["//visibility:public"],
    deps = [
        ":image_to_tensor_calculator_cc_proto",
        ":image_to_tensor_converter",
@@ -749,6 +955,7 @@ cc_library(
 cc_library(
    name = "image_to_tensor_calculator_gpu_deps",
    visibility = ["//visibility:private"],
    deps = selects.with_or({
        "//mediapipe:android": [
            ":image_to_tensor_converter_gl_buffer",
@@ -772,7 +979,6 @@ cc_library(
 mediapipe_proto_library(
    name = "image_to_tensor_calculator_proto",
    srcs = ["image_to_tensor_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -818,6 +1024,7 @@ cc_test(
        "//mediapipe/framework/port:opencv_imgcodecs",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/util:image_test_utils",
        "@com_google_absl//absl/flags:flag",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/strings",
@@ -834,7 +1041,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":image_to_tensor_utils",
        "//mediapipe/framework/formats:image",
@@ -854,7 +1060,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":image_to_tensor_converter",
        ":image_to_tensor_utils",
@@ -874,6 +1079,7 @@ cc_library(
    name = "image_to_tensor_converter_gl_buffer",
    srcs = ["image_to_tensor_converter_gl_buffer.cc"],
    hdrs = ["image_to_tensor_converter_gl_buffer.h"],
    visibility = ["//visibility:private"],
    deps = ["//mediapipe/framework:port"] + selects.with_or({
        "//mediapipe:apple": [],
        "//conditions:default": [
@@ -907,6 +1113,7 @@ cc_library(
    name = "image_to_tensor_converter_gl_texture",
    srcs = ["image_to_tensor_converter_gl_texture.cc"],
    hdrs = ["image_to_tensor_converter_gl_texture.h"],
    visibility = ["//visibility:private"],
    deps = ["//mediapipe/framework:port"] + select({
        "//mediapipe/gpu:disable_gpu": [],
        "//conditions:default": [
@@ -931,6 +1138,7 @@ cc_library(
    name = "image_to_tensor_converter_gl_utils",
    srcs = ["image_to_tensor_converter_gl_utils.cc"],
    hdrs = ["image_to_tensor_converter_gl_utils.h"],
    visibility = ["//visibility:private"],
    deps = ["//mediapipe/framework:port"] + select({
        "//mediapipe/gpu:disable_gpu": [],
        "//conditions:default": [
@@ -960,6 +1168,7 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:private"],
    deps = ["//mediapipe/framework:port"] + select({
        "//mediapipe:apple": [
            ":image_to_tensor_converter",
@@ -967,7 +1176,6 @@ cc_library(
            "//mediapipe/gpu:MPPMetalHelper",
            "@com_google_absl//absl/strings",
            "//mediapipe/framework:calculator_framework",
            "//mediapipe/framework/formats:rect_cc_proto",
            "//mediapipe/framework/formats:tensor",
            "//mediapipe/framework/port:ret_check",
            "//mediapipe/framework/port:status",
@@ -985,22 +1193,41 @@ cc_library(
    name = "image_to_tensor_utils",
    srcs = ["image_to_tensor_utils.cc"],
    hdrs = ["image_to_tensor_utils.h"],
-    visibility = ["//visibility:public"],
+    copts = select({
        "//mediapipe:apple": [
            "-x objective-c++",
            "-fobjc-arc",  # enable reference-counting
        ],
        "//conditions:default": [],
    }),
    deps = [
        ":image_to_tensor_calculator_cc_proto",
        "@com_google_absl//absl/status",
        "@com_google_absl//absl/types:optional",
        "//mediapipe/framework/api2:packet",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:image",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:statusor",
-        "@com_google_absl//absl/types:optional",
+        "//mediapipe/gpu:gpu_origin_cc_proto",
-    ],
+    ] + select({
        "//mediapipe/gpu:disable_gpu": [],
        "//conditions:default": ["//mediapipe/gpu:gpu_buffer"],
    }),
 )
 cc_test(
    name = "image_to_tensor_utils_test",
    srcs = ["image_to_tensor_utils_test.cc"],
    deps = [
        ":image_to_tensor_calculator_cc_proto",
        ":image_to_tensor_utils",
        "//mediapipe/framework/formats:rect_cc_proto",
        "//mediapipe/framework/port:gtest_main",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/port:status",
    ],
 )
@@ -1015,7 +1242,6 @@ selects.config_setting_group(
 mediapipe_proto_library(
    name = "tensors_to_segmentation_calculator_proto",
    srcs = ["tensors_to_segmentation_calculator.proto"],
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_options_proto",
        "//mediapipe/framework:calculator_proto",
@@ -1033,7 +1259,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        ":tensors_to_segmentation_calculator_cc_proto",
        "@com_google_absl//absl/strings:str_format",
@@ -1046,9 +1271,9 @@ cc_library(
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:port",
        "//mediapipe/gpu:gpu_origin_cc_proto",
        "//mediapipe/util:resource_util",
        "@org_tensorflow//tensorflow/lite:framework",
        "//mediapipe/gpu:gpu_origin_cc_proto",
        "//mediapipe/framework/port:statusor",
    ] + selects.with_or({
        "//mediapipe/gpu:disable_gpu": [],
@@ -1091,7 +1316,6 @@ cc_library(
        ],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
    deps = [
        "//mediapipe/framework:calculator_context",
        "//mediapipe/framework:calculator_framework",
@@ -12,9 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <math.h>
 #include <algorithm>
 #include <cmath>
 #include <cstring>
 #include <memory>
 #include <string>
@@ -26,6 +25,7 @@
 #include "absl/status/statusor.h"
 #include "absl/strings/str_format.h"
 #include "audio/dsp/resampler_q.h"
 #include "audio/dsp/window_functions.h"
 #include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/packet.h"
@@ -34,19 +34,61 @@
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/time_series_util.h"
 #include "pffft.h"
 namespace mediapipe {
 namespace api2 {
 namespace {
 using Options = ::mediapipe::AudioToTensorCalculatorOptions;
 using DftTensorFormat = Options::DftTensorFormat;
 using FlushMode = Options::FlushMode;
 std::vector<float> HannWindow(int window_size, bool sqrt_hann) {
  std::vector<float> hann_window(window_size);
  audio_dsp::HannWindow().GetPeriodicSamples(window_size, &hann_window);
  if (sqrt_hann) {
    absl::c_transform(hann_window, hann_window.begin(),
                      [](double x) { return std::sqrt(x); });
  }
  return hann_window;
 }
 // PFFFT only supports transforms for inputs of length N of the form
 // N = (2^a)*(3^b)*(5^c) where b >=0 and c >= 0 and a >= 5 for the real FFT.
 bool IsValidFftSize(int size) {
  if (size <= 0) {
    return false;
  }
  constexpr int kFactors[] = {2, 3, 5};
  int factorization[] = {0, 0, 0};
  int n = static_cast<int>(size);
  for (int i = 0; i < 3; ++i) {
    while (n % kFactors[i] == 0) {
      n = n / kFactors[i];
      ++factorization[i];
    }
  }
  return factorization[0] >= 5 && n == 1;
 }
 }  // namespace
 // Converts audio buffers into tensors, possibly with resampling, buffering
 // and framing, according to specified inputs and options. All input audio
 // buffers will be first resampled from the input sample rate to the target
 // sample rate if they are not equal. The resampled audio data (with the
 // buffered samples from the previous runs in the streaming mode) will be broken
-// into fixed-sized, possibly overlapping frames. Finally, all frames will be
+// into fixed-sized, possibly overlapping frames. If the calculator is not asked
-// converted to and outputted as MediaPipe Tensors. The last output tensor will
+// to perform fft (the fft_size is not set in the calculator options), all
-// be zero-padding if the remaining samples are insufficient.
+// frames will be converted to and outputted as MediaPipe Tensors. The last
 // output tensor will be zero-padding if the remaining samples are insufficient.
 // Otherwise, when the fft_size is set and valid, the calculator will perform
 // fft on the fixed-sized audio frames, the complex DFT results will be
 // converted to and outputted as 2D MediaPipe float Tensors where the first
 // rows are the DFT real parts and the second rows are the DFT imagery parts.
 //
 // This calculator assumes that the input timestamps refer to the first
 // sample in each Matrix. The output timestamps follow this same convention.
@@ -86,11 +128,15 @@ namespace api2 {
 // Outputs:
 //   TENSORS - std::vector<Tensor>
 //     Vector containing a single Tensor that represents a fix-sized audio
-//     frame.
+//     frame or the complex DFT results.
 //   TIMESTAMPS - std::vector<Timestamp> @Optional
 //     Vector containing the output timestamps emitted by the current Process()
 //     invocation. In the non-streaming mode, the vector contains all of the
 //     output timestamps for an input audio buffer.
 //   DC_AND_NYQUIST - std::pair<float, float> @Optional.
 //     A pair of dc component and nyquist component. Only can be connected when
 //     the calculator performs fft (the fft_size is set in the calculator
 //     options).
 //
 // Example:
 // node {
@@ -116,12 +162,14 @@ class AudioToTensorCalculator : public Node {
  // such as sample rate.
  static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
  static constexpr Output<std::pair<float, float>>::Optional kDcAndNyquistOut{
      "DC_AND_NYQUIST"};
  // A vector of the output timestamps emitted by the current Process()
  // invocation. The packet timestamp is the last emitted timestamp.
  static constexpr Output<std::vector<Timestamp>>::Optional kTimestampsOut{
      "TIMESTAMPS"};
  MEDIAPIPE_NODE_CONTRACT(kAudioIn, kAudioSampleRateIn, kTensorsOut,
-                          kTimestampsOut);
+                          kDcAndNyquistOut, kTimestampsOut);
  static absl::Status UpdateContract(CalculatorContract* cc);
  absl::Status Open(CalculatorContext* cc);
@@ -138,6 +186,11 @@ class AudioToTensorCalculator : public Node {
  int frame_step_;
  bool stream_mode_;
  bool check_inconsistent_timestamps_;
  int padding_samples_before_;
  int padding_samples_after_;
  FlushMode flush_mode_;
  DftTensorFormat dft_tensor_format_;
  Timestamp initial_timestamp_ = Timestamp::Unstarted();
  int64 cumulative_input_samples_ = 0;
  Timestamp next_output_timestamp_ = Timestamp::Unstarted();
@@ -151,22 +204,33 @@ class AudioToTensorCalculator : public Node {
  Matrix sample_buffer_;
  int processed_buffer_cols_ = 0;
  // The internal state of the FFT library.
  PFFFT_Setup* fft_state_ = nullptr;
  int fft_size_ = 0;
  std::vector<float> fft_window_;
  std::vector<float, Eigen::aligned_allocator<float>> fft_input_buffer_;
  // pffft requires memory to work with to avoid using the stack.
  std::vector<float, Eigen::aligned_allocator<float>> fft_workplace_;
  std::vector<float, Eigen::aligned_allocator<float>> fft_output_;
  absl::Status ProcessStreamingData(CalculatorContext* cc, const Matrix& input);
  absl::Status ProcessNonStreamingData(CalculatorContext* cc,
                                       const Matrix& input);
  absl::Status SetupStreamingResampler(double input_sample_rate_);
  void AppendToSampleBuffer(Matrix buffer_to_append);
  void AppendZerosToSampleBuffer(int num_samples);
  absl::StatusOr<std::vector<Tensor>> ConvertToTensor(
-      const Matrix& frame_to_convert);
+      const Matrix& block, std::vector<int> tensor_dims);
-  absl::Status OutputTensors(const Matrix& buffer, bool should_flush,
+  absl::Status OutputTensor(const Matrix& block, Timestamp timestamp,
                            CalculatorContext* cc);
  absl::Status ProcessBuffer(const Matrix& buffer, bool should_flush,
                             CalculatorContext* cc);
 };
 absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
-  const auto& options =
+  const auto& options = cc->Options<Options>();
      cc->Options<mediapipe::AudioToTensorCalculatorOptions>();
  if (!options.has_num_channels() || !options.has_num_samples() ||
      !options.has_target_sample_rate()) {
    return absl::InvalidArgumentError(
@@ -174,13 +238,21 @@ absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
        "`num_channels`, `num_samples`, and `target_sample_rate`.");
  }
  if (options.stream_mode()) {
-    // Explicitly disables tiemstamp offset to disallow the timestamp bound
+    // Explicitly disables timestamp offset to disallow the timestamp bound
    // from the input streams to be propagated to the output streams.
    // In the streaming mode, the output timestamp bound is based on
    // next_output_timestamp_, which can be smaller than the current input
    // timestamps.
    cc->SetTimestampOffset(TimestampDiff::Unset());
  }
  if (options.padding_samples_before() < 0 ||
      options.padding_samples_after() < 0) {
    return absl::InvalidArgumentError("Negative zero padding unsupported");
  }
  if (options.flush_mode() != Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX &&
      options.flush_mode() != Options::PROCEED_AS_USUAL) {
    return absl::InvalidArgumentError("Unsupported flush mode");
  }
  return absl::OkStatus();
 }
@@ -202,6 +274,10 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
    check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
    sample_buffer_.resize(num_channels_, Eigen::NoChange);
  }
  padding_samples_before_ = options.padding_samples_before();
  padding_samples_after_ = options.padding_samples_after();
  dft_tensor_format_ = options.dft_tensor_format();
  flush_mode_ = options.flush_mode();
  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
            !kAudioIn(cc).Header().IsEmpty())
@@ -217,6 +293,25 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
      source_sample_rate_ = input_header.sample_rate();
    }
  }
  AppendZerosToSampleBuffer(padding_samples_before_);
  if (options.has_fft_size()) {
    RET_CHECK(IsValidFftSize(options.fft_size()))
        << "FFT size must be of the form fft_size = (2^a)*(3^b)*(5^c) where b "
           ">=0 and c >= 0 and a >= 5, the requested fft size is "
        << options.fft_size();
    RET_CHECK_EQ(1, num_channels_)
        << "Currently only support applying FFT on mono channel.";
    fft_size_ = options.fft_size();
    fft_state_ = pffft_new_setup(fft_size_, PFFFT_REAL);
    fft_window_ = HannWindow(fft_size_, /* sqrt_hann = */ false);
    fft_input_buffer_.resize(fft_size_);
    fft_workplace_.resize(fft_size_);
    fft_output_.resize(fft_size_);
  } else {
    RET_CHECK(!kDcAndNyquistOut(cc).IsConnected())
        << "The DC_AND_NYQUIST output stream can only be connected when the "
           "calculator outputs fft tensors";
  }
  return absl::OkStatus();
 }
@@ -262,7 +357,12 @@ absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
    resampler_->Flush(&resampled_buffer);
    AppendToSampleBuffer(std::move(resampled_buffer));
  }
-  return OutputTensors(sample_buffer_, /*should_flush=*/true, cc);
+  AppendZerosToSampleBuffer(padding_samples_after_);
  MP_RETURN_IF_ERROR(ProcessBuffer(sample_buffer_, /*should_flush=*/true, cc));
  if (fft_state_) {
    pffft_destroy_setup(fft_state_);
  }
  return absl::OkStatus();
 }
 absl::Status AudioToTensorCalculator::ProcessStreamingData(
@@ -303,7 +403,7 @@ absl::Status AudioToTensorCalculator::ProcessStreamingData(
    }
  }
-  MP_RETURN_IF_ERROR(OutputTensors(sample_buffer_, /*should_flush=*/false, cc));
+  MP_RETURN_IF_ERROR(ProcessBuffer(sample_buffer_, /*should_flush=*/false, cc));
  // Removes the processed samples from the global sample buffer.
  sample_buffer_ = Matrix(sample_buffer_.rightCols(sample_buffer_.cols() -
                                                   processed_buffer_cols_ - 1));
@@ -323,9 +423,9 @@ absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
        input_frame);
    Eigen::Map<const Matrix> matrix_mapping(resampled.data(), num_channels_,
                                            resampled.size() / num_channels_);
-    return OutputTensors(matrix_mapping, /*should_flush=*/true, cc);
+    return ProcessBuffer(matrix_mapping, /*should_flush=*/true, cc);
  }
-  return OutputTensors(input_frame, /*should_flush=*/true, cc);
+  return ProcessBuffer(input_frame, /*should_flush=*/true, cc);
 }
 absl::Status AudioToTensorCalculator::SetupStreamingResampler(
@@ -344,6 +444,16 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
  return absl::OkStatus();
 }
 void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
  CHECK_GE(num_samples, 0);  // Ensured by `UpdateContract`.
  if (num_samples == 0) {
    return;
  }
  sample_buffer_.conservativeResize(Eigen::NoChange,
                                    sample_buffer_.cols() + num_samples);
  sample_buffer_.rightCols(num_samples).setZero();
 }
 void AudioToTensorCalculator::AppendToSampleBuffer(Matrix buffer_to_append) {
  sample_buffer_.conservativeResize(
      Eigen::NoChange, sample_buffer_.cols() + buffer_to_append.cols());
@@ -351,49 +461,118 @@ void AudioToTensorCalculator::AppendToSampleBuffer(Matrix buffer_to_append) {
 }
 absl::StatusOr<std::vector<Tensor>> AudioToTensorCalculator::ConvertToTensor(
-    const Matrix& frame_to_convert) {
+    const Matrix& block, std::vector<int> tensor_dims) {
-  Tensor tensor(Tensor::ElementType::kFloat32,
+  Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape(tensor_dims));
                Tensor::Shape({num_channels_, num_samples_}));
  auto buffer_view = tensor.GetCpuWriteView();
-  if (frame_to_convert.size() < num_channels_ * num_samples_) {
+  int total_size = 1;
  for (int dim : tensor_dims) {
    total_size *= dim;
  }
  if (block.size() < total_size) {
    std::memset(buffer_view.buffer<float>(), 0, tensor.bytes());
  }
-  std::memcpy(buffer_view.buffer<float>(), frame_to_convert.data(),
+  std::memcpy(buffer_view.buffer<float>(), block.data(),
-              frame_to_convert.size() * sizeof(float));
+              block.size() * sizeof(float));
  std::vector<Tensor> tensor_vector;
  tensor_vector.push_back(std::move(tensor));
  return tensor_vector;
 }
-absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
+absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
                                                   Timestamp timestamp,
                                                   CalculatorContext* cc) {
  std::vector<Tensor> output_tensor;
  if (fft_state_) {
    Eigen::VectorXf time_series_data =
        Eigen::VectorXf::Map(block.data(), block.size());
    //  Window on input audio prior to FFT.
    std::transform(time_series_data.begin(), time_series_data.end(),
                   fft_window_.begin(), fft_input_buffer_.begin(),
                   std::multiplies<float>());
    pffft_transform_ordered(fft_state_, fft_input_buffer_.data(),
                            fft_output_.data(), fft_workplace_.data(),
                            PFFFT_FORWARD);
    if (kDcAndNyquistOut(cc).IsConnected()) {
      kDcAndNyquistOut(cc).Send(std::make_pair(fft_output_[0], fft_output_[1]),
                                timestamp);
    }
    switch (dft_tensor_format_) {
      case Options::WITH_NYQUIST: {
        Matrix fft_output_matrix =
            Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
        fft_output_matrix.conservativeResize(Eigen::NoChange, fft_size_);
        // The last two elements are Nyquist component.
        fft_output_matrix(fft_size_ - 2) = fft_output_[1];  // Nyquist real part
        fft_output_matrix(fft_size_ - 1) = 0.0f;  // Nyquist imagery part
        ASSIGN_OR_RETURN(output_tensor, ConvertToTensor(fft_output_matrix,
                                                        {2, fft_size_ / 2}));
        break;
      }
      case Options::WITH_DC_AND_NYQUIST: {
        Matrix fft_output_matrix =
            Eigen::Map<const Matrix>(fft_output_.data(), 1, fft_size_);
        fft_output_matrix.conservativeResize(Eigen::NoChange, fft_size_ + 2);
        fft_output_matrix(1) = 0.0f;  // DC imagery part.
        // The last two elements are  Nyquist component.
        fft_output_matrix(fft_size_) = fft_output_[1];  // Nyquist real part
        fft_output_matrix(fft_size_ + 1) = 0.0f;        // Nyquist imagery part
        ASSIGN_OR_RETURN(
            output_tensor,
            ConvertToTensor(fft_output_matrix, {2, (fft_size_ + 2) / 2}));
        break;
      }
      case Options::WITHOUT_DC_AND_NYQUIST: {
        Matrix fft_output_matrix =
            Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
        ASSIGN_OR_RETURN(
            output_tensor,
            ConvertToTensor(fft_output_matrix, {2, (fft_size_ - 2) / 2}));
        break;
      }
      default:
        return absl::InvalidArgumentError("Unsupported dft tensor format.");
    }
  } else {
    ASSIGN_OR_RETURN(output_tensor,
                     ConvertToTensor(block, {num_channels_, num_samples_}));
  }
  kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
  return absl::OkStatus();
 }
 absl::Status AudioToTensorCalculator::ProcessBuffer(const Matrix& buffer,
                                                    bool should_flush,
                                                    CalculatorContext* cc) {
  const bool should_flush_at_timestamp_max =
      stream_mode_ && should_flush &&
      flush_mode_ == Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX;
  int next_frame_first_col = 0;
  std::vector<Timestamp> timestamps;
-  while ((!stream_mode_ || !should_flush) &&
+  if (!should_flush_at_timestamp_max) {
-         next_frame_first_col + num_samples_ <= buffer.cols()) {
+    while (next_frame_first_col + num_samples_ <= buffer.cols()) {
-    ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
+      MP_RETURN_IF_ERROR(OutputTensor(
-                                             0, next_frame_first_col,
+          buffer.block(0, next_frame_first_col, num_channels_, num_samples_),
-                                             num_channels_, num_samples_)));
+          next_output_timestamp_, cc));
-    kTensorsOut(cc).Send(std::move(output_tensor), next_output_timestamp_);
+      timestamps.push_back(next_output_timestamp_);
-    timestamps.push_back(next_output_timestamp_);
+      next_output_timestamp_ += round(frame_step_ / target_sample_rate_ *
-    next_output_timestamp_ += round(frame_step_ / target_sample_rate_ *
+                                      Timestamp::kTimestampUnitsPerSecond);
-                                    Timestamp::kTimestampUnitsPerSecond);
+      next_frame_first_col += frame_step_;
-    next_frame_first_col += frame_step_;
+    }
  }
  if (should_flush && next_frame_first_col < buffer.cols()) {
    ASSIGN_OR_RETURN(auto output_tensor,
                     ConvertToTensor(buffer.block(
                         0, next_frame_first_col, num_channels_,
                         std::min(num_samples_,
                                  (int)buffer.cols() - next_frame_first_col))));
    // In the streaming mode, the flush happens in Close() and a packet at
    // Timestamp::Max() will be emitted. In the non-streaming mode, each
    // Process() invocation will process the entire buffer completely.
-    Timestamp timestamp =
+    Timestamp timestamp = should_flush_at_timestamp_max
-        stream_mode_ ? Timestamp::Max() : next_output_timestamp_;
+                              ? Timestamp::Max()
                              : next_output_timestamp_;
    MP_RETURN_IF_ERROR(OutputTensor(
        buffer.block(
            0, next_frame_first_col, num_channels_,
            std::min(num_samples_, (int)buffer.cols() - next_frame_first_col)),
        timestamp, cc));
    timestamps.push_back(timestamp);
    kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
  }
  if (kTimestampsOut(cc).IsConnected()) {
    Timestamp timestamp = timestamps.back();
@@ -44,4 +44,41 @@ message AudioToTensorCalculatorOptions {
  // Set to false to disable checks for jitter in timestamp values. Useful with
  // live audio input.
  optional bool check_inconsistent_timestamps = 6 [default = true];
  // Size of the fft in number of bins. If set, the calculator outputs fft
  // tensors.
  optional int64 fft_size = 7;
  // The amount of padding samples to add before the audio after resampling.
  // Note that the timestamps shift. Currently, only zero padding is supported.
  optional int64 padding_samples_before = 8;
  // The amount of padding samples to add after the audio after resampling.
  // Currently, only zero padding is supported.
  optional int64 padding_samples_after = 9;
  // Determines the "flushing" behavior in stream mode.
  enum FlushMode {
    // Unspecified (causes an error). Won't be used because of the default.
    NONE = 0;
    // Emit a packet with the entire remainder at `Timestamp::Max`.
    ENTIRE_TAIL_AT_TIMESTAMP_MAX = 1;
    // Continue emitting framed packets with relevant timestamps.
    PROCEED_AS_USUAL = 2;
  }
  optional FlushMode flush_mode = 10 [default = ENTIRE_TAIL_AT_TIMESTAMP_MAX];
  enum DftTensorFormat {
    DFT_TENSOR_FORMAT_UNKNOWN = 0;
    // The output dft tensor without dc and nyquist components.
    WITHOUT_DC_AND_NYQUIST = 1;
    // The output dft tensor contains the nyquist component as the last
    // two values.
    WITH_NYQUIST = 2;
    // The output dft tensor contains the dc component as the first two values
    // and the nyquist component as the last two values.
    WITH_DC_AND_NYQUIST = 3;
  }
  optional DftTensorFormat dft_tensor_format = 11 [default = WITH_NYQUIST];
 }
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <cmath>
 #include <memory>
 #include <string>
 #include <vector>
 #include "absl/strings/substitute.h"
 #include "audio/dsp/resampler_q.h"
 #include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -32,6 +32,14 @@
 namespace mediapipe {
 namespace {
 using ::testing::Not;
 using Options = ::mediapipe::AudioToTensorCalculatorOptions;
 using FlushMode = Options::FlushMode;
 int DivideRoundedUp(int dividend, int divisor) {
  return (dividend + divisor - 1) / divisor;
 }
 std::unique_ptr<Matrix> CreateTestMatrix(int num_channels, int num_samples,
                                         int timestamp) {
  auto matrix = std::make_unique<Matrix>(num_channels, num_samples);
@@ -292,16 +300,17 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
    num_iterations_ = num_iterations;
  }
-  int GetExpectedNumOfSamples() {
+  int GetExpectedNumOfSamples() { return output_sample_buffer_->cols(); }
    Matrix* expected_matrix =
        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
    return expected_matrix->cols();
  }
  void Run(int num_samples, int num_overlapping_samples,
-           double resampling_factor) {
+           double resampling_factor, int padding_before = 0,
           int padding_after = 0, bool expect_init_error = false) {
    double input_sample_rate = 10000;
    double target_sample_rate = input_sample_rate * resampling_factor;
    FlushMode flush_mode = (padding_before != 0 || padding_after != 0)
                               ? Options::PROCEED_AS_USUAL
                               : Options::ENTIRE_TAIL_AT_TIMESTAMP_MAX;
    auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
        absl::Substitute(R"(
        input_stream: "audio"
@@ -319,16 +328,25 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
              num_overlapping_samples: $1
              target_sample_rate: $2
              stream_mode:true
              padding_samples_before: $3
              padding_samples_after: $4
              flush_mode: $5
            }
          }
        }
        )",
                         /*$0=*/num_samples, /*$1=*/num_overlapping_samples,
-                         /*$2=*/target_sample_rate));
+                         /*$2=*/target_sample_rate, /*$3=*/padding_before,
                         /*$4=*/padding_after, /*$5=*/flush_mode));
    tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
    // Run the graph.
-    MP_ASSERT_OK(graph_.Initialize(graph_config));
+    const absl::Status init_status = graph_.Initialize(graph_config);
    if (expect_init_error) {
      EXPECT_THAT(init_status, Not(IsOk()));
      return;
    }
    MP_ASSERT_OK(init_status);
    MP_ASSERT_OK(graph_.StartRun({}));
    for (int i = 0; i < num_iterations_; ++i) {
      Timestamp input_timestamp(Timestamp::kTimestampUnitsPerSecond * i);
@@ -345,8 +363,18 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
    }
    MP_ASSERT_OK(graph_.CloseAllInputStreams());
    MP_ASSERT_OK(graph_.WaitUntilIdle());
-    if (resampling_factor != 1) {
+    if (resampling_factor == 1) {
-      resampled_buffer_ = ResampleBuffer(*sample_buffer_, resampling_factor);
+      output_sample_buffer_ = std::make_unique<Matrix>(*sample_buffer_);
    } else {
      output_sample_buffer_ =
          ResampleBuffer(*sample_buffer_, resampling_factor);
    }
    if (padding_before != 0 || padding_after != 0) {
      Matrix padded = Matrix::Zero(
          2, padding_before + output_sample_buffer_->cols() + padding_after);
      padded.block(0, padding_before, 2, output_sample_buffer_->cols()) =
          *output_sample_buffer_;
      output_sample_buffer_->swap(padded);
    }
  }
@@ -372,15 +400,13 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
    auto buffer = output_tensor.GetCpuReadView().buffer<float>();
    int num_values = output_tensor.shape().num_elements();
    std::vector<float> output_floats(buffer, buffer + num_values);
    Matrix* expected_matrix =
        resampled_buffer_ ? resampled_buffer_.get() : sample_buffer_.get();
    for (int i = 0; i < num_values; ++i) {
-      if (i + sample_offset >= expected_matrix->size()) {
+      if (i + sample_offset >= output_sample_buffer_->size()) {
        EXPECT_FLOAT_EQ(output_floats[i], 0);
      } else {
        EXPECT_NEAR(output_floats[i],
-                    expected_matrix->coeff((i + sample_offset) % 2,
+                    output_sample_buffer_->coeff((i + sample_offset) % 2,
-                                           (i + sample_offset) / 2),
+                                                 (i + sample_offset) / 2),
                    0.001)
            << "i=" << i << ", sample_offset=" << sample_offset
            << ", packet index=" << index;
@@ -391,7 +417,8 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
  // Fully close graph at end, otherwise calculator+tensors are destroyed
  // after calling WaitUntilDone().
-  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
+  absl::Status TryCloseGraph() { return graph_.WaitUntilDone(); }
  void CloseGraph() { MP_EXPECT_OK(TryCloseGraph()); }
 private:
  int input_buffer_num_samples_ = 10;
@@ -399,7 +426,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
  CalculatorGraph graph_;
  std::vector<Packet> tensors_packets_;
  std::unique_ptr<Matrix> sample_buffer_;
-  std::unique_ptr<Matrix> resampled_buffer_;
+  std::unique_ptr<Matrix> output_sample_buffer_;
 };
 TEST_F(AudioToTensorCalculatorStreamingModeTest,
@@ -408,7 +435,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest,
      /*resampling_factor=*/1.0f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/10,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 5),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 5),
      /*timestamp_interval=*/500,
      /*output_last_at_close=*/false);
  CloseGraph();
@@ -419,7 +446,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputRemainingInCloseMethod) {
      /*resampling_factor=*/1.0f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/12,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 6),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 6),
      /*timestamp_interval=*/600,
      /*output_last_at_close=*/true);
  CloseGraph();
@@ -431,7 +458,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, OutputOverlappingFp32Tensors) {
      /*resampling_factor=*/1.0f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/16,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 8),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 8),
      /*timestamp_interval=*/800,
      /*output_last_at_close=*/true);
  CloseGraph();
@@ -443,7 +470,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, Downsampling) {
      /*resampling_factor=*/0.5f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/512,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 256),
      /*timestamp_interval=*/51200,
      /*output_last_at_close=*/true);
  CloseGraph();
@@ -455,7 +482,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, DownsamplingWithOverlapping) {
      /*resampling_factor=*/0.5f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/384,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
      /*timestamp_interval=*/38400,
      /*output_last_at_close=*/true);
  CloseGraph();
@@ -467,7 +494,7 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, Upsampling) {
      /*resampling_factor=*/2.0f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/512,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 256),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 256),
      /*timestamp_interval=*/12800,
      /*output_last_at_close=*/true);
  CloseGraph();
@@ -479,12 +506,33 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest, UpsamplingWithOverlapping) {
      /*resampling_factor=*/2.0f);
  CheckTensorsOutputPackets(
      /*sample_offset=*/384,
-      /*num_packets=*/std::ceil((float)GetExpectedNumOfSamples() / 192),
+      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
      /*timestamp_interval=*/9600,
      /*output_last_at_close=*/true);
  CloseGraph();
 }
 TEST_F(AudioToTensorCalculatorStreamingModeTest,
       UpsamplingWithOverlappingAndPadding) {
  SetInputBufferNumSamplesPerChannel(1024);
  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
      /*resampling_factor=*/2.0f, /*padding_before=*/13, /*padding_after=*/999);
  CheckTensorsOutputPackets(
      /*sample_offset=*/384,
      /*num_packets=*/DivideRoundedUp(GetExpectedNumOfSamples(), 192),
      /*timestamp_interval=*/9600,
      /*output_last_at_close=*/false);
  CloseGraph();
 }
 TEST_F(AudioToTensorCalculatorStreamingModeTest, NegativePaddingUnsupported) {
  SetInputBufferNumSamplesPerChannel(1024);
  Run(/*num_samples=*/256, /*num_overlapping_samples=*/64,
      /*resampling_factor=*/2.0f, /*padding_before=*/13, /*padding_after=*/-3,
      /*expect_init_error=*/true);
  EXPECT_THAT(TryCloseGraph(), Not(IsOk()));
 }
 TEST_F(AudioToTensorCalculatorStreamingModeTest,
       OnlyOutputInCloseIfNoSufficientSamples) {
  SetNumIterations(1);
@@ -498,5 +546,122 @@ TEST_F(AudioToTensorCalculatorStreamingModeTest,
  CloseGraph();
 }
 class AudioToTensorCalculatorFftTest : public ::testing::Test {
 protected:
  // Creates an audio matrix containing a single sample of 1.0 at a specified
  // offset.
  std::unique_ptr<Matrix> CreateImpulseSignalData(int64 num_samples,
                                                  int impulse_offset_idx) {
    Matrix impulse = Matrix::Zero(1, num_samples);
    impulse(0, impulse_offset_idx) = 1.0;
    return std::make_unique<Matrix>(std::move(impulse));
  }
  void ConfigGraph(int num_channels, int num_samples,
                   int num_overlapping_samples, double sample_rate,
                   int fft_size) {
    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
        absl::Substitute(R"(
        input_stream: "audio"
        input_stream: "sample_rate"
        output_stream: "tensors"
        output_stream: "dc_and_nyquist"
        node {
          calculator: "AudioToTensorCalculator"
          input_stream: "AUDIO:audio"
          input_stream: "SAMPLE_RATE:sample_rate"
          output_stream: "TENSORS:tensors"
          output_stream: "DC_AND_NYQUIST:dc_and_nyquist"
          options {
            [mediapipe.AudioToTensorCalculatorOptions.ext] {
              num_channels: $0
              num_samples: $1
              num_overlapping_samples: $2
              target_sample_rate: $3
              fft_size: $4
            }
          }
        }
        )",
                         /*$0=*/num_channels,
                         /*$1=*/num_samples,
                         /*$2=*/num_overlapping_samples,
                         /*$3=*/sample_rate, /*$4=*/fft_size));
    std::vector<Packet> tensors_packets;
    tool::AddVectorSink("tensors", &graph_config_, &tensors_packets_);
    std::vector<Packet> dc_and_nyquist_packets;
    tool::AddVectorSink("dc_and_nyquist", &graph_config_,
                        &dc_and_nyquist_packets_);
  }
  void RunGraph(std::unique_ptr<Matrix> input_data, double sample_rate) {
    MP_ASSERT_OK(graph_.Initialize(graph_config_));
    MP_ASSERT_OK(graph_.StartRun({}));
    MP_ASSERT_OK(graph_.AddPacketToInputStream(
        "sample_rate", MakePacket<double>(sample_rate).At(Timestamp(0))));
    MP_ASSERT_OK(graph_.AddPacketToInputStream(
        "audio", MakePacket<Matrix>(*input_data).At(Timestamp(0))));
    MP_ASSERT_OK(graph_.CloseAllInputStreams());
    MP_ASSERT_OK(graph_.WaitUntilIdle());
    ASSERT_EQ(tensors_packets_.size(), dc_and_nyquist_packets_.size());
  }
  // Fully close graph at end, otherwise calculator+tensors are destroyed
  // after calling WaitUntilDone().
  void CloseGraph() { MP_EXPECT_OK(graph_.WaitUntilDone()); }
  std::vector<Packet> tensors_packets_;
  std::vector<Packet> dc_and_nyquist_packets_;
  CalculatorGraphConfig graph_config_;
  CalculatorGraph graph_;
 };
 TEST_F(AudioToTensorCalculatorFftTest, TestInvalidFftSize) {
  ConfigGraph(1, 320, 160, 16000, 103);
  MP_ASSERT_OK(graph_.Initialize(graph_config_));
  MP_ASSERT_OK(graph_.StartRun({}));
  auto status = graph_.WaitUntilIdle();
  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
  EXPECT_THAT(status.message(),
              ::testing::HasSubstr("FFT size must be of the form"));
 }
 TEST_F(AudioToTensorCalculatorFftTest, TestInvalidNumChannels) {
  ConfigGraph(3, 320, 160, 16000, 256);
  MP_ASSERT_OK(graph_.Initialize(graph_config_));
  MP_ASSERT_OK(graph_.StartRun({}));
  auto status = graph_.WaitUntilIdle();
  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
  EXPECT_THAT(
      status.message(),
      ::testing::HasSubstr("only support applying FFT on mono channel"));
 }
 TEST_F(AudioToTensorCalculatorFftTest, TestImpulseSignal) {
  constexpr double sample_rate = 16000;
  ConfigGraph(1, 320, 160, sample_rate, 320);
  RunGraph(CreateImpulseSignalData(320, 160), sample_rate);
  for (int i = 0; i < tensors_packets_.size(); ++i) {
    const auto& tensors = tensors_packets_[i].Get<std::vector<Tensor>>();
    ASSERT_EQ(1, tensors.size());
    const Tensor& output_tensor =
        tensors_packets_[0].Get<std::vector<Tensor>>()[0];
    auto* buffer = output_tensor.GetCpuReadView().buffer<float>();
    int num_values = output_tensor.shape().num_elements();
    const std::vector<float> output_floats(buffer, buffer + num_values);
    // Impulse signal should have (approximately) const power across all
    // frequency bins.
    const auto& pair =
        dc_and_nyquist_packets_[i].Get<std::pair<float, float>>();
    EXPECT_FLOAT_EQ(pair.first, 1.0f);
    EXPECT_FLOAT_EQ(pair.second, 1.0f);
    for (int j = 0; j < num_values / 2; ++j) {
      std::complex<float> cf(output_floats[j * 2], output_floats[j * 2 + 1]);
      EXPECT_FLOAT_EQ(std::norm(cf), 1.0f);
    }
  }
  CloseGraph();
 }
 }  // namespace
 }  // namespace mediapipe
@@ -0,0 +1,251 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/bert_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/tasks/cc/core/utils.h"
 #include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
 #include "mediapipe/tasks/cc/text/tokenizers/tokenizer.h"
 #include "mediapipe/tasks/cc/text/tokenizers/tokenizer_utils.h"
 #include "mediapipe/tasks/metadata/metadata_schema_generated.h"
 namespace mediapipe {
 namespace api2 {
 using ::mediapipe::tasks::core::FindTensorIndexByMetadataName;
 using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
 constexpr int kNumInputTensorsForBert = 3;
 constexpr int kTokenizerProcessUnitIndex = 0;
 constexpr absl::string_view kInputIdsTensorName = "ids";
 constexpr absl::string_view kInputMasksTensorName = "mask";
 constexpr absl::string_view kSegmentIdsTensorName = "segment_ids";
 constexpr absl::string_view kClassifierToken = "[CLS]";
 constexpr absl::string_view kSeparatorToken = "[SEP]";
 // Preprocesses input text into three int32 input tensors for a BERT model using
 // a tokenizer.
 // The associated BERT model is expected to contain input tensors with names:
 //
 // Tensor           | Metadata Name
 // ---------------- | --------------
 // IDs              | "ids"
 // Segment IDs      | "segment_ids"
 // Mask             | "mask"
 //
 // This calculator will return an error if the model does not have three input
 // tensors or if the tensors do not have names corresponding to the above
 // metadata names in some order. Additional details regarding these input
 // tensors are given in the Calculator "Outputs" section below.
 //
 // This calculator is currently configured for the TextClassifier Task but it
 // will eventually be generalized for other Text Tasks.
 // TODO: Handle preprocessing for other Text Tasks too.
 //
 // Inputs:
 //   TEXT - std::string
 //     The input text.
 // Side Inputs:
 //   METADATA_EXTRACTOR - ModelMetadataExtractor
 //     The metadata extractor for the BERT model. Used to determine the order of
 //     the three input Tensors for the BERT model and to extract the metadata to
 //     construct the tokenizer.
 //
 // Outputs:
 //   TENSORS - std::vector<Tensor>
 //     Vector containing the three input Tensors for the BERT model:
 //       (1): the token ids of the tokenized input string. A classifier token
 //            ("[CLS]") will be prepended to the input tokens and a separator
 //            token ("[SEP]") will be appended to the input tokens.
 //       (2): the segment ids, which are all 0 for now but will have different
 //            values to distinguish between different sentences in the input
 //            text for other Text tasks.
 //       (3): the input mask ids, which are 1 at each of the input token indices
 //            and 0 elsewhere.
 //     The Tensors will have size equal to the max sequence length for the BERT
 //     model.
 //
 // Example:
 // node {
 //   calculator: "BertPreprocessorCalculator"
 //   input_stream: "TEXT:text"
 //   input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
 //   output_stream: "TENSORS:tensors"
 //   options {
 //     [mediapipe.BertPreprocessorCalculatorOptions.ext] {
 //       bert_max_seq_len: 128
 //     }
 //   }
 // }
 class BertPreprocessorCalculator : public Node {
 public:
  static constexpr Input<std::string> kTextIn{"TEXT"};
  static constexpr SideInput<ModelMetadataExtractor> kMetadataExtractorSideIn{
      "METADATA_EXTRACTOR"};
  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
  MEDIAPIPE_NODE_CONTRACT(kTextIn, kMetadataExtractorSideIn, kTensorsOut);
  static absl::Status UpdateContract(CalculatorContract* cc);
  absl::Status Open(CalculatorContext* cc) override;
  absl::Status Process(CalculatorContext* cc) override;
 private:
  std::unique_ptr<tasks::text::tokenizers::Tokenizer> tokenizer_;
  // The max sequence length accepted by the BERT model.
  int bert_max_seq_len_ = 2;
  // Indices of the three input tensors for the BERT model. They should form the
  // set {0, 1, 2}.
  int input_ids_tensor_index_ = 0;
  int segment_ids_tensor_index_ = 1;
  int input_masks_tensor_index_ = 2;
  // Applies `tokenizer_` to the `input_text` to generate a vector of tokens.
  // This util prepends "[CLS]" and appends "[SEP]" to the input tokens and
  // clips the vector of tokens to have length at most `bert_max_seq_len_`.
  std::vector<std::string> TokenizeInputText(absl::string_view input_text);
  // Processes the `input_tokens` to generate the three input tensors for the
  // BERT model.
  std::vector<Tensor> GenerateInputTensors(
      const std::vector<std::string>& input_tokens);
 };
 absl::Status BertPreprocessorCalculator::UpdateContract(
    CalculatorContract* cc) {
  const auto& options =
      cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
  RET_CHECK(options.has_bert_max_seq_len()) << "bert_max_seq_len is required";
  RET_CHECK_GE(options.bert_max_seq_len(), 2)
      << "bert_max_seq_len must be at least 2";
  return absl::OkStatus();
 }
 absl::Status BertPreprocessorCalculator::Open(CalculatorContext* cc) {
  const ModelMetadataExtractor* metadata_extractor =
      &kMetadataExtractorSideIn(cc).Get();
  const tflite::ProcessUnit* tokenizer_metadata =
      metadata_extractor->GetInputProcessUnit(kTokenizerProcessUnitIndex);
  ASSIGN_OR_RETURN(tokenizer_,
                   tasks::text::tokenizers::CreateTokenizerFromProcessUnit(
                       tokenizer_metadata, metadata_extractor));
  auto* input_tensors_metadata = metadata_extractor->GetInputTensorMetadata();
  input_ids_tensor_index_ = FindTensorIndexByMetadataName(
      input_tensors_metadata, kInputIdsTensorName);
  segment_ids_tensor_index_ = FindTensorIndexByMetadataName(
      input_tensors_metadata, kSegmentIdsTensorName);
  input_masks_tensor_index_ = FindTensorIndexByMetadataName(
      input_tensors_metadata, kInputMasksTensorName);
  absl::flat_hash_set<int> tensor_indices = {input_ids_tensor_index_,
                                             segment_ids_tensor_index_,
                                             input_masks_tensor_index_};
  if (tensor_indices != absl::flat_hash_set<int>({0, 1, 2})) {
    return absl::InvalidArgumentError(absl::Substitute(
        "Input tensor indices form the set {$0, $1, $2} rather than {0, 1, 2}",
        input_ids_tensor_index_, segment_ids_tensor_index_,
        input_masks_tensor_index_));
  }
  const auto& options =
      cc->Options<mediapipe::BertPreprocessorCalculatorOptions>();
  bert_max_seq_len_ = options.bert_max_seq_len();
  return absl::OkStatus();
 }
 absl::Status BertPreprocessorCalculator::Process(CalculatorContext* cc) {
  kTensorsOut(cc).Send(
      GenerateInputTensors(TokenizeInputText(kTextIn(cc).Get())));
  return absl::OkStatus();
 }
 std::vector<std::string> BertPreprocessorCalculator::TokenizeInputText(
    absl::string_view input_text) {
  std::string processed_input = std::string(input_text);
  absl::AsciiStrToLower(&processed_input);
  tasks::text::tokenizers::TokenizerResult tokenizer_result =
      tokenizer_->Tokenize(processed_input);
  // Offset by 2 to account for [CLS] and [SEP]
  int input_tokens_size =
      std::min(bert_max_seq_len_,
               static_cast<int>(tokenizer_result.subwords.size()) + 2);
  std::vector<std::string> input_tokens;
  input_tokens.reserve(input_tokens_size);
  input_tokens.push_back(std::string(kClassifierToken));
  for (int i = 0; i < input_tokens_size - 2; ++i) {
    input_tokens.push_back(std::move(tokenizer_result.subwords[i]));
  }
  input_tokens.push_back(std::string(kSeparatorToken));
  return input_tokens;
 }
 std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
    const std::vector<std::string>& input_tokens) {
  std::vector<int32_t> input_ids(bert_max_seq_len_, 0);
  std::vector<int32_t> segment_ids(bert_max_seq_len_, 0);
  std::vector<int32_t> input_masks(bert_max_seq_len_, 0);
  // Convert tokens back into ids and set mask
  for (int i = 0; i < input_tokens.size(); ++i) {
    tokenizer_->LookupId(input_tokens[i], &input_ids[i]);
    input_masks[i] = 1;
  }
  //                           |<--------bert_max_seq_len_--------->|
  // input_ids                 [CLS] s1  s2...  sn [SEP]  0  0...  0
  // segment_ids                 0    0   0...  0    0    0  0...  0
  // input_masks                 1    1   1...  1    1    0  0...  0
  std::vector<Tensor> input_tensors;
  input_tensors.reserve(kNumInputTensorsForBert);
  for (int i = 0; i < kNumInputTensorsForBert; ++i) {
    input_tensors.push_back(
        {Tensor::ElementType::kInt32, Tensor::Shape({bert_max_seq_len_})});
  }
  std::memcpy(input_tensors[input_ids_tensor_index_]
                  .GetCpuWriteView()
                  .buffer<int32_t>(),
              input_ids.data(), input_ids.size() * sizeof(int32_t));
  std::memcpy(input_tensors[segment_ids_tensor_index_]
                  .GetCpuWriteView()
                  .buffer<int32_t>(),
              segment_ids.data(), segment_ids.size() * sizeof(int32_t));
  std::memcpy(input_tensors[input_masks_tensor_index_]
                  .GetCpuWriteView()
                  .buffer<int32_t>(),
              input_masks.data(), input_masks.size() * sizeof(int32_t));
  return input_tensors;
 }
 MEDIAPIPE_REGISTER_NODE(BertPreprocessorCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,29 @@
 /* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 message BertPreprocessorCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional BertPreprocessorCalculatorOptions ext = 462509271;
  }
  // The maximum input sequence length for the calculator's BERT model.
  optional int32 bert_max_seq_len = 1;
 }
@@ -0,0 +1,154 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <memory>
 #include <sstream>
 #include <string>
 #include <utility>
 #include <vector>
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/tasks/cc/core/utils.h"
 #include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
 namespace mediapipe {
 namespace {
 using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
 using ::testing::ElementsAreArray;
 constexpr int kNumInputTensorsForBert = 3;
 constexpr int kBertMaxSeqLen = 128;
 constexpr absl::string_view kTestModelPath =
    "mediapipe/tasks/testdata/text/bert_text_classifier.tflite";
 absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
    absl::string_view text, absl::string_view model_path) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
      absl::Substitute(R"(
        input_stream: "text"
        output_stream: "tensors"
        node {
          calculator: "BertPreprocessorCalculator"
          input_stream: "TEXT:text"
          input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
          output_stream: "TENSORS:tensors"
          options {
            [mediapipe.BertPreprocessorCalculatorOptions.ext] {
              bert_max_seq_len: $0
            }
          }
        }
      )",
                       kBertMaxSeqLen));
  std::vector<Packet> output_packets;
  tool::AddVectorSink("tensors", &graph_config, &output_packets);
  std::string model_buffer = tasks::core::LoadBinaryContent(model_path.data());
  ASSIGN_OR_RETURN(std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
                   ModelMetadataExtractor::CreateFromModelBuffer(
                       model_buffer.data(), model_buffer.size()));
  // Run the graph.
  CalculatorGraph graph;
  MP_RETURN_IF_ERROR(graph.Initialize(
      graph_config,
      {{"metadata_extractor",
        MakePacket<ModelMetadataExtractor>(std::move(*metadata_extractor))}}));
  MP_RETURN_IF_ERROR(graph.StartRun({}));
  MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
      "text", MakePacket<std::string>(text).At(Timestamp(0))));
  MP_RETURN_IF_ERROR(graph.WaitUntilIdle());
  if (output_packets.size() != 1) {
    return absl::InvalidArgumentError(absl::Substitute(
        "output_packets has size $0, expected 1", output_packets.size()));
  }
  const std::vector<Tensor>& tensor_vec =
      output_packets[0].Get<std::vector<Tensor>>();
  if (tensor_vec.size() != kNumInputTensorsForBert) {
    return absl::InvalidArgumentError(
        absl::Substitute("tensor_vec has size $0, expected $1",
                         tensor_vec.size(), kNumInputTensorsForBert));
  }
  std::vector<std::vector<int>> results;
  for (int i = 0; i < kNumInputTensorsForBert; i++) {
    const Tensor& tensor = tensor_vec[i];
    if (tensor.element_type() != Tensor::ElementType::kInt32) {
      return absl::InvalidArgumentError("Expected tensor element type kInt32");
    }
    auto* buffer = tensor.GetCpuReadView().buffer<int>();
    std::vector<int> buffer_view(buffer, buffer + kBertMaxSeqLen);
    results.push_back(buffer_view);
  }
  MP_RETURN_IF_ERROR(graph.CloseAllPacketSources());
  MP_RETURN_IF_ERROR(graph.WaitUntilDone());
  return results;
 }
 TEST(BertPreprocessorCalculatorTest, TextClassifierWithBertModel) {
  std::vector<std::vector<int>> expected_result = {
      {101, 2009, 1005, 1055, 1037, 11951, 1998, 2411, 12473, 4990, 102}};
  // segment_ids
  expected_result.push_back(std::vector(kBertMaxSeqLen, 0));
  // input_masks
  expected_result.push_back(std::vector(expected_result[0].size(), 1));
  expected_result[2].resize(kBertMaxSeqLen);
  // padding input_ids
  expected_result[0].resize(kBertMaxSeqLen);
  MP_ASSERT_OK_AND_ASSIGN(
      std::vector<std::vector<int>> processed_tensor_values,
      RunBertPreprocessorCalculator(
          "it's a charming and often affecting journey", kTestModelPath));
  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
 }
 TEST(BertPreprocessorCalculatorTest, LongInput) {
  std::stringstream long_input;
  long_input
      << "it's a charming and often affecting journey and this is a long";
  for (int i = 0; i < kBertMaxSeqLen; ++i) {
    long_input << " long";
  }
  long_input << " movie review";
  std::vector<std::vector<int>> expected_result = {
      {101, 2009, 1005, 1055, 1037, 11951, 1998, 2411, 12473, 4990, 1998, 2023,
       2003, 1037}};
  // "long" id
  expected_result[0].resize(kBertMaxSeqLen - 1, 2146);
  // "[SEP]" id
  expected_result[0].push_back(102);
  // segment_ids
  expected_result.push_back(std::vector(kBertMaxSeqLen, 0));
  // input_masks
  expected_result.push_back(std::vector(kBertMaxSeqLen, 1));
  MP_ASSERT_OK_AND_ASSIGN(
      std::vector<std::vector<int>> processed_tensor_values,
      RunBertPreprocessorCalculator(long_input.str(), kTestModelPath));
  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
 }
 }  // namespace
 }  // namespace mediapipe
@@ -0,0 +1,165 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <algorithm>
 #include <memory>
 #include <utility>
 #include "absl/status/status.h"
 #include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 namespace mediapipe {
 namespace api2 {
 namespace {
 constexpr char kInputTensorsTag[] = "INPUT_TENSORS";
 constexpr char kFeedbackTensorsTag[] = "FEEDBACK_TENSORS";
 constexpr char kOutputTensorsTag[] = "TENSORS";
 using Tensors = std::vector<Tensor>;
 }  // namespace
 // FeedbackTensorsCalculator groups the input and the feedback (typically
 // recurrent neural network cell state output tensors from the previous run)
 // tensor vectors as the input tensor vector for the next recurrent model cell
 // inference. On the first step, the feedback tensor is filled with zeros to
 // jumpstart the loop.
 class FeedbackTensorsCalculator : public Node {
 public:
  static constexpr Input<Tensors> kFeedbackTensorsIn{kFeedbackTensorsTag};
  static constexpr Input<Tensors> kInputTensorsIn{kInputTensorsTag};
  static constexpr Output<Tensors> kTensorsOut{kOutputTensorsTag};
  MEDIAPIPE_NODE_CONTRACT(kFeedbackTensorsIn, kInputTensorsIn, kTensorsOut);
  static absl::Status GetContract(CalculatorContract* cc) {
    cc->SetProcessTimestampBounds(true);
    return absl::OkStatus();
  }
  absl::Status Open(CalculatorContext* cc) override {
    const auto& options =
        cc->Options<mediapipe::FeedbackTensorsCalculatorOptions>();
    const auto& shape_dims = options.feedback_tensor_shape().dims();
    feedback_tensor_shape_.dims.assign(shape_dims.begin(), shape_dims.end());
    feedback_tensor_size_ = feedback_tensor_shape_.num_elements();
    num_feedback_tensors_ = options.num_feedback_tensors();
    feedback_tensors_location_ = options.location();
    return absl::OkStatus();
  }
  absl::Status Process(CalculatorContext* cc) override {
    if (feedback_tensors_location_ ==
        mediapipe::FeedbackTensorsCalculatorOptions::NONE) {
      kTensorsOut(cc).Send(kInputTensorsIn(cc).packet().As<Tensors>());
      return absl::OkStatus();
    }
    std::vector<Tensor> outputs;
    switch (feedback_tensors_location_) {
      case mediapipe::FeedbackTensorsCalculatorOptions::PREPENDED:
        MP_RETURN_IF_ERROR(AddFeedbackTensors(cc, outputs));
        MP_RETURN_IF_ERROR(AddInputTensors(cc, outputs));
        break;
      case mediapipe::FeedbackTensorsCalculatorOptions::APPENDED:
        MP_RETURN_IF_ERROR(AddInputTensors(cc, outputs));
        MP_RETURN_IF_ERROR(AddFeedbackTensors(cc, outputs));
        break;
      default:
        return absl::InvalidArgumentError(
            "Unsupported feedback tensors location");
    }
    kTensorsOut(cc).Send(std::move(outputs));
    return absl::OkStatus();
  }
 private:
  absl::Status AddInputTensors(CalculatorContext* cc,
                               std::vector<Tensor>& outputs) {
    absl::StatusOr<std::unique_ptr<std::vector<Tensor>>> input_tensors =
        cc->Inputs()
            .Tag(kInputTensorsTag)
            .Value()
            .Consume<std::vector<Tensor>>();
    if (!input_tensors.ok()) {
      return absl::InternalError("The input tensors packet is not consumable");
    }
    RET_CHECK(*input_tensors);
    std::vector<Tensor>& inputs = **input_tensors;
    outputs.insert(outputs.end(), std::make_move_iterator(inputs.begin()),
                   std::make_move_iterator(inputs.end()));
    return absl::OkStatus();
  }
  absl::Status AddFeedbackTensors(CalculatorContext* cc,
                                  std::vector<Tensor>& outputs) {
    if (first_run_) {
      for (int index = 0; index < num_feedback_tensors_; ++index) {
        Tensor initial_feedback_tensor(Tensor::ElementType::kFloat32,
                                       feedback_tensor_shape_);
        float* data = initial_feedback_tensor.GetCpuWriteView().buffer<float>();
        std::fill_n(data, feedback_tensor_size_, 0.0f);
        outputs.push_back(std::move(initial_feedback_tensor));
      }
      first_run_ = false;
      return absl::OkStatus();
    }
    if (num_feedback_tensors_ != kFeedbackTensorsIn(cc)->size()) {
      return absl::InvalidArgumentError(
          "The number of tensors fed back differs from the configuration");
    }
    absl::StatusOr<std::unique_ptr<std::vector<Tensor>>> feedback_tensors =
        cc->Inputs()
            .Tag(kFeedbackTensorsTag)
            .Value()
            .Consume<std::vector<Tensor>>();
    if (!feedback_tensors.ok()) {
      return absl::InternalError(
          "The feedback tensors packet is not consumable");
    }
    RET_CHECK(*feedback_tensors);
    std::vector<Tensor>& feedbacks = **feedback_tensors;
    for (const auto& feedback : feedbacks) {
      if (feedback.shape().dims != feedback_tensor_shape_.dims) {
        return absl::InvalidArgumentError(
            "The shape of a tensor fed back differs from the configuration");
      }
    }
    outputs.insert(outputs.end(), std::make_move_iterator(feedbacks.begin()),
                   std::make_move_iterator(feedbacks.end()));
    return absl::OkStatus();
  }
  Tensor::Shape feedback_tensor_shape_;
  int num_feedback_tensors_ = 0;
  mediapipe::FeedbackTensorsCalculatorOptions::FeedbackTensorsLocation
      feedback_tensors_location_;
  int feedback_tensor_size_ = 0;
  bool first_run_ = true;
 };
 MEDIAPIPE_REGISTER_NODE(FeedbackTensorsCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,47 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 message FeedbackTensorsCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional FeedbackTensorsCalculatorOptions ext = 474496252;
  }
  // Represents the dimensions of a tensor starting from the outermost size.
  message TensorShape {
    repeated int32 dims = 1 [packed = true];
  }
  // The shape of the feedback tensors to add.
  optional TensorShape feedback_tensor_shape = 1;
  // The number of the feedback tensors to add.
  optional int32 num_feedback_tensors = 2 [default = 1];
  enum FeedbackTensorsLocation {
    // The feedback tensors will not be added.
    NONE = 0;
    // The feedback tensors will be added before the input tensors.
    PREPENDED = 1;
    // The feedback tensors will be added after the input tensors.
    APPENDED = 2;
  }
  // Determines the location of the feedback tensor(s) in the output vector.
  optional FeedbackTensorsLocation location = 3 [default = APPENDED];
 }
@@ -0,0 +1,389 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <functional>
 #include <initializer_list>
 #include <memory>
 #include <utility>
 #include <vector>
 #include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/timestamp.h"
 namespace mediapipe {
 namespace {
 using ::mediapipe::CalculatorGraphConfig;
 using ::testing::ElementsAreArray;
 using ::testing::Not;
 using Tensors = std::vector<Tensor>;
 template <typename T>
 struct TensorElementType {
  static constexpr Tensor::ElementType value = Tensor::ElementType::kNone;
 };
 template <>
 struct TensorElementType<float> {
  static constexpr Tensor::ElementType value = Tensor::ElementType::kFloat32;
 };
 template <>
 struct TensorElementType<std::int8_t> {
  static constexpr Tensor::ElementType value = Tensor::ElementType::kInt8;
 };
 template <>
 struct TensorElementType<std::uint8_t> {
  static constexpr Tensor::ElementType value = Tensor::ElementType::kUInt8;
 };
 template <>
 struct TensorElementType<std::int32_t> {
  static constexpr Tensor::ElementType value = Tensor::ElementType::kInt32;
 };
 template <typename T>
 Tensor MakeTensor(std::initializer_list<int> shape,
                  std::initializer_list<T> values) {
  Tensor tensor(TensorElementType<T>::value, shape);
  CHECK_EQ(values.size(), tensor.shape().num_elements())
      << "The size of `values` is incompatible with `shape`";
  absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
  return tensor;
 }
 template <typename T>
 void ValidateTensor(const Tensor& tensor,
                    const std::vector<int>& expected_shape,
                    const std::vector<T>& expected_values) {
  ASSERT_EQ(tensor.element_type(), TensorElementType<T>::value);
  EXPECT_EQ(tensor.shape().dims, expected_shape);
  EXPECT_EQ(tensor.shape().num_elements(), expected_values.size());
  auto* tensor_buffer = tensor.GetCpuReadView().buffer<T>();
  const std::vector<T> tensor_values(
      tensor_buffer, tensor_buffer + tensor.shape().num_elements());
  EXPECT_THAT(tensor_values, ElementsAreArray(expected_values));
 }
 TEST(FeedbackTensorsCalculatorTest, AppendsFeedback) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
    input_stream: "input"
    input_stream: "feedback"
    node {
      calculator: "FeedbackTensorsCalculator"
      input_stream: "INPUT_TENSORS:input"
      input_stream: "FEEDBACK_TENSORS:feedback"
      output_stream: "TENSORS:output"
      options: {
        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
          feedback_tensor_shape: { dims: 2 dims: 3 }
          location: APPENDED
        }
      }
    }
  )pb");
  std::vector<Packet> output_packets;
  tool::AddVectorSink("output", &graph_config, &output_packets);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(graph_config));
  MP_ASSERT_OK(graph.StartRun({}));
  auto initial_input_tensors = std::make_unique<Tensors>();
  initial_input_tensors->push_back(
      MakeTensor<std::int32_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
  // At the beginning, the loopback packet with the model feedback is missing.
  // The calculator has to assume it's all-zero with the shape from the options.
  auto later_input_tensors = std::make_unique<Tensors>();
  later_input_tensors->push_back(
      MakeTensor<std::int32_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
  auto later_feedback_tensors = std::make_unique<Tensors>();
  later_feedback_tensors->push_back(
      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
  MP_ASSERT_OK(graph.CloseAllInputStreams())
      << "Couldn't close the graph inputs";
  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
  ASSERT_EQ(output_packets.size(), 2);
  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
  ASSERT_EQ(initial_combined_tensors.size(), 2);
  ValidateTensor<std::int32_t>(initial_combined_tensors[0],
                               /*expected_shape=*/{2, 4},
                               /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
  // The initial feedback is zero.
  ValidateTensor<float>(initial_combined_tensors[1], /*expected_shape=*/{2, 3},
                        /*expected_values=*/{0.f, 0.f, 0.f, 0.f, 0.f, 0.f});
  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
  ASSERT_EQ(later_combined_tensors.size(), 2);
  ValidateTensor<std::int32_t>(later_combined_tensors[0],
                               /*expected_shape=*/{2, 4},
                               /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
  // Afterwards, the provided feedback is passed through.
  ValidateTensor<float>(
      later_combined_tensors[1], /*expected_shape=*/{2, 3},
      /*expected_values=*/{-1.f, -2.f, -3.f, -4.f, -5.f, -6.f});
 }
 TEST(FeedbackTensorsCalculatorTest, PrependsFeedback) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
    input_stream: "input"
    input_stream: "feedback"
    node {
      calculator: "FeedbackTensorsCalculator"
      input_stream: "INPUT_TENSORS:input"
      input_stream: "FEEDBACK_TENSORS:feedback"
      output_stream: "TENSORS:output"
      options: {
        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
          feedback_tensor_shape: { dims: 3 dims: 2 }
          location: PREPENDED
        }
      }
    }
  )pb");
  std::vector<Packet> output_packets;
  tool::AddVectorSink("output", &graph_config, &output_packets);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(graph_config));
  MP_ASSERT_OK(graph.StartRun({}));
  auto initial_input_tensors = std::make_unique<Tensors>();
  initial_input_tensors->push_back(
      MakeTensor<std::int8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
  // At the beginning, the loopback packet with the model feedback is missing.
  // The calculator has to assume it's all-zero with the shape from the options.
  auto later_input_tensors = std::make_unique<Tensors>();
  later_input_tensors->push_back(
      MakeTensor<std::int8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
  auto later_feedback_tensors = std::make_unique<Tensors>();
  later_feedback_tensors->push_back(
      MakeTensor({3, 2}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
  MP_ASSERT_OK(graph.CloseAllInputStreams())
      << "Couldn't close the graph inputs";
  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
  ASSERT_EQ(output_packets.size(), 2);
  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
  ASSERT_EQ(initial_combined_tensors.size(), 2);
  // The initial feedback is zero.
  ValidateTensor<float>(initial_combined_tensors[0], /*expected_shape=*/{3, 2},
                        /*expected_values=*/{0.f, 0.f, 0.f, 0.f, 0.f, 0.f});
  ValidateTensor<std::int8_t>(initial_combined_tensors[1],
                              /*expected_shape=*/{2, 4},
                              /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
  ASSERT_EQ(later_combined_tensors.size(), 2);
  // Afterwards, the provided feedback is passed through.
  ValidateTensor<float>(
      later_combined_tensors[0], /*expected_shape=*/{3, 2},
      /*expected_values=*/{-1.f, -2.f, -3.f, -4.f, -5.f, -6.f});
  ValidateTensor<std::int8_t>(later_combined_tensors[1],
                              /*expected_shape=*/{2, 4},
                              /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
 }
 TEST(FeedbackTensorsCalculatorTest, NoFeedback) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
    input_stream: "input"
    input_stream: "feedback"
    node {
      calculator: "FeedbackTensorsCalculator"
      input_stream: "INPUT_TENSORS:input"
      input_stream: "FEEDBACK_TENSORS:feedback"
      output_stream: "TENSORS:output"
      options: {
        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
          feedback_tensor_shape: { dims: 3 dims: 4 }
          location: NONE
        }
      }
    }
  )pb");
  std::vector<Packet> output_packets;
  tool::AddVectorSink("output", &graph_config, &output_packets);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(graph_config));
  MP_ASSERT_OK(graph.StartRun({}));
  auto initial_input_tensors = std::make_unique<Tensors>();
  initial_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
  // At the beginning, the loopback packet with the model feedback is missing.
  auto later_input_tensors = std::make_unique<Tensors>();
  later_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
  // This feedback should be ignored due to `location: NONE`.
  auto later_feedback_tensors = std::make_unique<Tensors>();
  later_feedback_tensors->push_back(
      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
  MP_ASSERT_OK(graph.CloseAllInputStreams())
      << "Couldn't close the graph inputs";
  MP_ASSERT_OK(graph.WaitUntilDone()) << "Couldn't finalize the graph run";
  ASSERT_EQ(output_packets.size(), 2);
  const Tensors& initial_combined_tensors = output_packets[0].Get<Tensors>();
  ASSERT_EQ(initial_combined_tensors.size(), 1);
  ValidateTensor<std::uint8_t>(initial_combined_tensors[0],
                               /*expected_shape=*/{2, 4},
                               /*expected_values=*/{1, 2, 3, 4, 5, 6, 7, 8});
  // No feedback due to `location: NONE`.
  const Tensors& later_combined_tensors = output_packets[1].Get<Tensors>();
  ASSERT_EQ(later_combined_tensors.size(), 1);
  ValidateTensor<std::uint8_t>(later_combined_tensors[0],
                               /*expected_shape=*/{2, 4},
                               /*expected_values=*/{8, 7, 6, 5, 4, 3, 2, 1});
 }
 TEST(FeedbackTensorsCalculatorTest, ChecksTensorNumber) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
    input_stream: "input"
    input_stream: "feedback"
    node {
      calculator: "FeedbackTensorsCalculator"
      input_stream: "INPUT_TENSORS:input"
      input_stream: "FEEDBACK_TENSORS:feedback"
      output_stream: "TENSORS:output"
      options: {
        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
          num_feedback_tensors: 2
          feedback_tensor_shape: { dims: 2 dims: 3 }
          location: PREPENDED
        }
      }
    }
  )pb");
  std::vector<Packet> output_packets;
  tool::AddVectorSink("output", &graph_config, &output_packets);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(graph_config));
  MP_ASSERT_OK(graph.StartRun({}));
  auto initial_input_tensors = std::make_unique<Tensors>();
  initial_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
  // At the beginning, the loopback packet with the model feedback is missing.
  auto later_input_tensors = std::make_unique<Tensors>();
  later_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
  // This feedback should be ignored due to `location: NONE`.
  auto later_feedback_tensors = std::make_unique<Tensors>();
  later_feedback_tensors->push_back(
      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
  MP_ASSERT_OK(graph.CloseAllInputStreams())
      << "Couldn't close the graph inputs";
  EXPECT_THAT(graph.WaitUntilDone(), Not(IsOk()))
      << "Tensor number mismatch missed";
 }
 TEST(FeedbackTensorsCalculatorTest, ChecksShape) {
  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
    input_stream: "input"
    input_stream: "feedback"
    node {
      calculator: "FeedbackTensorsCalculator"
      input_stream: "INPUT_TENSORS:input"
      input_stream: "FEEDBACK_TENSORS:feedback"
      output_stream: "TENSORS:output"
      options: {
        [mediapipe.FeedbackTensorsCalculatorOptions.ext] {
          feedback_tensor_shape: { dims: 3 dims: 4 }
          location: APPENDED
        }
      }
    }
  )pb");
  std::vector<Packet> output_packets;
  tool::AddVectorSink("output", &graph_config, &output_packets);
  CalculatorGraph graph;
  MP_ASSERT_OK(graph.Initialize(graph_config));
  MP_ASSERT_OK(graph.StartRun({}));
  auto initial_input_tensors = std::make_unique<Tensors>();
  initial_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {1, 2, 3, 4, 5, 6, 7, 8}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(initial_input_tensors.release()).At(Timestamp(1))));
  // At the beginning, the loopback packet with the model feedback is missing.
  auto later_input_tensors = std::make_unique<Tensors>();
  later_input_tensors->push_back(
      MakeTensor<std::uint8_t>({2, 4}, {8, 7, 6, 5, 4, 3, 2, 1}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "input", Adopt(later_input_tensors.release()).At(Timestamp(2))));
  // This feedback should be ignored due to `location: NONE`.
  auto later_feedback_tensors = std::make_unique<Tensors>();
  later_feedback_tensors->push_back(
      MakeTensor({2, 3}, {-1.f, -2.f, -3.f, -4.f, -5.f, -6.f}));
  MP_ASSERT_OK(graph.AddPacketToInputStream(
      "feedback", Adopt(later_feedback_tensors.release()).At(Timestamp(2))));
  MP_ASSERT_OK(graph.CloseAllInputStreams())
      << "Couldn't close the graph inputs";
  EXPECT_THAT(graph.WaitUntilDone(), Not(IsOk()))
      << "Tensor shape mismatch missed";
 }
 }  // namespace
 }  // namespace mediapipe
@@ -54,13 +54,6 @@
 namespace mediapipe {
 namespace api2 {
 #if MEDIAPIPE_DISABLE_GPU
 // Just a placeholder to not have to depend on mediapipe::GpuBuffer.
 using GpuBuffer = AnyType;
 #else
 using GpuBuffer = mediapipe::GpuBuffer;
 #endif  // MEDIAPIPE_DISABLE_GPU
 // Converts image into Tensor, possibly with cropping, resizing and
 // normalization, according to specified inputs and options.
 //
@@ -141,42 +134,7 @@ class ImageToTensorCalculator : public Node {
    const auto& options =
        cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
-    RET_CHECK(options.has_output_tensor_float_range() ||
+    RET_CHECK_OK(ValidateOptionOutputDims(options));
              options.has_output_tensor_int_range() ||
              options.has_output_tensor_uint_range())
        << "Output tensor range is required.";
    if (options.has_output_tensor_float_range()) {
      RET_CHECK_LT(options.output_tensor_float_range().min(),
                   options.output_tensor_float_range().max())
          << "Valid output float tensor range is required.";
    }
    if (options.has_output_tensor_uint_range()) {
      RET_CHECK_LT(options.output_tensor_uint_range().min(),
                   options.output_tensor_uint_range().max())
          << "Valid output uint tensor range is required.";
      RET_CHECK_GE(options.output_tensor_uint_range().min(), 0)
          << "The minimum of the output uint tensor range must be "
             "non-negative.";
      RET_CHECK_LE(options.output_tensor_uint_range().max(), 255)
          << "The maximum of the output uint tensor range must be less than or "
             "equal to 255.";
    }
    if (options.has_output_tensor_int_range()) {
      RET_CHECK_LT(options.output_tensor_int_range().min(),
                   options.output_tensor_int_range().max())
          << "Valid output int tensor range is required.";
      RET_CHECK_GE(options.output_tensor_int_range().min(), -128)
          << "The minimum of the output int tensor range must be greater than "
             "or equal to -128.";
      RET_CHECK_LE(options.output_tensor_int_range().max(), 127)
          << "The maximum of the output int tensor range must be less than or "
             "equal to 127.";
    }
    RET_CHECK_GT(options.output_tensor_width(), 0)
        << "Valid output tensor width is required.";
    RET_CHECK_GT(options.output_tensor_height(), 0)
        << "Valid output tensor height is required.";
    RET_CHECK(kIn(cc).IsConnected() ^ kInGpu(cc).IsConnected())
        << "One and only one of IMAGE and IMAGE_GPU input is expected.";
@@ -198,21 +156,7 @@ class ImageToTensorCalculator : public Node {
  absl::Status Open(CalculatorContext* cc) {
    options_ = cc->Options<mediapipe::ImageToTensorCalculatorOptions>();
-    output_width_ = options_.output_tensor_width();
+    params_ = GetOutputTensorParams(options_);
    output_height_ = options_.output_tensor_height();
    is_float_output_ = options_.has_output_tensor_float_range();
    if (options_.has_output_tensor_uint_range()) {
      range_min_ =
          static_cast<float>(options_.output_tensor_uint_range().min());
      range_max_ =
          static_cast<float>(options_.output_tensor_uint_range().max());
    } else if (options_.has_output_tensor_int_range()) {
      range_min_ = static_cast<float>(options_.output_tensor_int_range().min());
      range_max_ = static_cast<float>(options_.output_tensor_int_range().max());
    } else {
      range_min_ = options_.output_tensor_float_range().min();
      range_max_ = options_.output_tensor_float_range().max();
    }
    return absl::OkStatus();
  }
@@ -242,9 +186,15 @@ class ImageToTensorCalculator : public Node {
      }
    }
-    ASSIGN_OR_RETURN(auto image, GetInputImage(cc));
+#if MEDIAPIPE_DISABLE_GPU
-    const Size size{image->width(), image->height()};
+    ASSIGN_OR_RETURN(auto image, GetInputImage(kIn(cc)));
-    RotatedRect roi = GetRoi(size.width, size.height, norm_rect);
+#else
    const bool is_input_gpu = kInGpu(cc).IsConnected();
    ASSIGN_OR_RETURN(auto image, is_input_gpu ? GetInputImage(kInGpu(cc))
                                              : GetInputImage(kIn(cc)));
 #endif  // MEDIAPIPE_DISABLE_GPU
    RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect);
    ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(),
                                          options_.output_tensor_height(),
                                          options_.keep_aspect_ratio(), &roi));
@@ -253,19 +203,24 @@ class ImageToTensorCalculator : public Node {
    }
    if (kOutMatrix(cc).IsConnected()) {
      std::array<float, 16> matrix;
-      GetRotatedSubRectToRectTransformMatrix(roi, size.width, size.height,
+      GetRotatedSubRectToRectTransformMatrix(
-                                             /*flip_horizontaly=*/false,
+          roi, image->width(), image->height(),
-                                             &matrix);
+          /*flip_horizontaly=*/false, &matrix);
      kOutMatrix(cc).Send(std::move(matrix));
    }
    // Lazy initialization of the GPU or CPU converter.
    MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, *image.get()));
-    ASSIGN_OR_RETURN(Tensor tensor,
+    Tensor::ElementType output_tensor_type =
-                     (image->UsesGpu() ? gpu_converter_ : cpu_converter_)
+        GetOutputTensorType(image->UsesGpu(), params_);
-                         ->Convert(*image, roi, {output_width_, output_height_},
+    Tensor tensor(output_tensor_type,
-                                   range_min_, range_max_));
+                  {1, params_.output_height, params_.output_width,
                   GetNumOutputChannels(*image)});
    MP_RETURN_IF_ERROR((image->UsesGpu() ? gpu_converter_ : cpu_converter_)
                           ->Convert(*image, roi, params_.range_min,
                                     params_.range_max,
                                     /*tensor_buffer_offset=*/0, tensor));
    auto result = std::make_unique<std::vector<Tensor>>();
    result->push_back(std::move(tensor));
@@ -275,65 +230,11 @@ class ImageToTensorCalculator : public Node {
  }
 private:
  bool DoesGpuInputStartAtBottom() {
    return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
  }
  BorderMode GetBorderMode() {
    switch (options_.border_mode()) {
      case mediapipe::
          ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED:
        return BorderMode::kReplicate;
      case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO:
        return BorderMode::kZero;
      case mediapipe::
          ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE:
        return BorderMode::kReplicate;
    }
  }
  Tensor::ElementType GetOutputTensorType() {
    if (is_float_output_) {
      return Tensor::ElementType::kFloat32;
    }
    if (range_min_ < 0) {
      return Tensor::ElementType::kInt8;
    } else {
      return Tensor::ElementType::kUInt8;
    }
  }
  absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
      CalculatorContext* cc) {
    if (kIn(cc).IsConnected()) {
      const auto& packet = kIn(cc).packet();
      return kIn(cc).Visit(
          [&packet](const mediapipe::Image&) {
            return SharedPtrWithPacket<mediapipe::Image>(packet);
          },
          [&packet](const mediapipe::ImageFrame&) {
            return std::make_shared<const mediapipe::Image>(
                std::const_pointer_cast<mediapipe::ImageFrame>(
                    SharedPtrWithPacket<mediapipe::ImageFrame>(packet)));
          });
    } else {  // if (kInGpu(cc).IsConnected())
 #if !MEDIAPIPE_DISABLE_GPU
      const GpuBuffer& input = *kInGpu(cc);
      // A shallow copy is okay since the resulting 'image' object is local in
      // Process(), and thus never outlives 'input'.
      return std::make_shared<const mediapipe::Image>(input);
 #else
      return absl::UnimplementedError(
          "GPU processing is disabled in build flags");
 #endif  // !MEDIAPIPE_DISABLE_GPU
    }
  }
  absl::Status InitConverterIfNecessary(CalculatorContext* cc,
                                        const Image& image) {
    // Lazy initialization of the GPU or CPU converter.
    if (image.UsesGpu()) {
-      if (!is_float_output_) {
+      if (!params_.is_float_output) {
        return absl::UnimplementedError(
            "ImageToTensorConverter for the input GPU image currently doesn't "
            "support quantization.");
@@ -341,18 +242,20 @@ class ImageToTensorCalculator : public Node {
      if (!gpu_converter_) {
 #if !MEDIAPIPE_DISABLE_GPU
 #if MEDIAPIPE_METAL_ENABLED
-        ASSIGN_OR_RETURN(gpu_converter_,
+        ASSIGN_OR_RETURN(
-                         CreateMetalConverter(cc, GetBorderMode()));
+            gpu_converter_,
            CreateMetalConverter(cc, GetBorderMode(options_.border_mode())));
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
        ASSIGN_OR_RETURN(gpu_converter_,
                         CreateImageToGlBufferTensorConverter(
-                             cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+                             cc, DoesGpuInputStartAtBottom(options_),
                             GetBorderMode(options_.border_mode())));
 #else
        if (!gpu_converter_) {
-          ASSIGN_OR_RETURN(
+          ASSIGN_OR_RETURN(gpu_converter_,
-              gpu_converter_,
+                           CreateImageToGlTextureTensorConverter(
-              CreateImageToGlTextureTensorConverter(
+                               cc, DoesGpuInputStartAtBottom(options_),
-                  cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
+                               GetBorderMode(options_.border_mode())));
        }
        if (!gpu_converter_) {
          return absl::UnimplementedError(
@@ -364,9 +267,10 @@ class ImageToTensorCalculator : public Node {
    } else {
      if (!cpu_converter_) {
 #if !MEDIAPIPE_DISABLE_OPENCV
-        ASSIGN_OR_RETURN(
+        ASSIGN_OR_RETURN(cpu_converter_,
-            cpu_converter_,
+                         CreateOpenCvConverter(
-            CreateOpenCvConverter(cc, GetBorderMode(), GetOutputTensorType()));
+                             cc, GetBorderMode(options_.border_mode()),
                             GetOutputTensorType(/*uses_gpu=*/false, params_)));
 #else
        LOG(FATAL) << "Cannot create image to tensor opencv converter since "
                      "MEDIAPIPE_DISABLE_OPENCV is defined.";
@@ -379,11 +283,7 @@ class ImageToTensorCalculator : public Node {
  std::unique_ptr<ImageToTensorConverter> gpu_converter_;
  std::unique_ptr<ImageToTensorConverter> cpu_converter_;
  mediapipe::ImageToTensorCalculatorOptions options_;
-  int output_width_ = 0;
+  OutputTensorParams params_;
  int output_height_ = 0;
  bool is_float_output_ = false;
  float range_min_ = 0.0f;
  float range_max_ = 1.0f;
 };
 MEDIAPIPE_REGISTER_NODE(ImageToTensorCalculator);
@@ -36,22 +36,17 @@
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/util/image_test_utils.h"
 namespace mediapipe {
 namespace {
-cv::Mat GetRgb(absl::string_view path) {
+constexpr char kTestDataDir[] =
-  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
+    "/mediapipe/calculators/tensor/testdata/"
-  cv::Mat rgb;
+    "image_to_tensor/";
  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGB);
  return rgb;
 }
-cv::Mat GetRgba(absl::string_view path) {
+std::string GetFilePath(absl::string_view filename) {
-  cv::Mat bgr = cv::imread(file::JoinPath("./", path));
+  return file::JoinPath("./", kTestDataDir, filename);
  cv::Mat rgb;
  cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA);
  return rgb;
 }
 // Image to tensor test template.
@@ -147,29 +142,34 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
  ASSERT_THAT(tensor_vec, testing::SizeIs(1));
  const Tensor& tensor = tensor_vec[0];
  const int channels = tensor.shape().dims[3];
  ASSERT_TRUE(channels == 1 || channels == 3);
  auto view = tensor.GetCpuReadView();
  cv::Mat tensor_mat;
  if (output_int_tensor) {
    if (range_min < 0) {
      EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kInt8);
-      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8SC3,
+      tensor_mat = cv::Mat(tensor_height, tensor_width,
                           channels == 1 ? CV_8SC1 : CV_8SC3,
                           const_cast<int8*>(view.buffer<int8>()));
    } else {
      EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kUInt8);
-      tensor_mat = cv::Mat(tensor_height, tensor_width, CV_8UC3,
+      tensor_mat = cv::Mat(tensor_height, tensor_width,
                           channels == 1 ? CV_8UC1 : CV_8UC3,
                           const_cast<uint8*>(view.buffer<uint8>()));
    }
  } else {
    EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32);
-    tensor_mat = cv::Mat(tensor_height, tensor_width, CV_32FC3,
+    tensor_mat = cv::Mat(tensor_height, tensor_width,
                         channels == 1 ? CV_32FC1 : CV_32FC3,
                         const_cast<float*>(view.buffer<float>()));
  }
  cv::Mat result_rgb;
  auto transformation =
      GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f).value();
-  tensor_mat.convertTo(result_rgb, CV_8UC3, transformation.scale,
+  tensor_mat.convertTo(result_rgb, channels == 1 ? CV_8UC1 : CV_8UC3,
-                       transformation.offset);
+                       transformation.scale, transformation.offset);
  cv::Mat diff;
  cv::absdiff(result_rgb, expected_result, diff);
@@ -185,17 +185,27 @@ void RunTestWithInputImagePacket(const Packet& input_image_packet,
  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
  if (image_channels == 4) {
    return ImageFormat::SRGBA;
  } else if (image_channels == 3) {
    return ImageFormat::SRGB;
  } else if (image_channels == 1) {
    return ImageFormat::GRAY8;
  }
  CHECK(false) << "Unsupported input image channles: " << image_channels;
 }
 Packet MakeImageFramePacket(cv::Mat input) {
-  ImageFrame input_image(
+  ImageFrame input_image(GetImageFormat(input.channels()), input.cols,
-      input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
+                         input.rows, input.step, input.data, [](uint8*) {});
      input.cols, input.rows, input.step, input.data, [](uint8*) {});
  return MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0));
 }
 Packet MakeImagePacket(cv::Mat input) {
  mediapipe::Image input_image(std::make_shared<mediapipe::ImageFrame>(
-      input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB,
+      GetImageFormat(input.channels()), input.cols, input.rows, input.step,
-      input.cols, input.rows, input.step, input.data, [](uint8*) {}));
+      input.data, [](uint8*) {}));
  return MakePacket<mediapipe::Image>(std::move(input_image)).At(Timestamp(0));
 }
@@ -237,15 +247,12 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) {
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(0);
-  RunTest(
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-      GetRgb("/mediapipe/calculators/"
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect.png")),
-             "tensor/testdata/image_to_tensor/input.jpg"),
+          /*float_ranges=*/{{0.0f, 1.0f}},
-      GetRgb("/mediapipe/calculators/"
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
-             "tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"),
+          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
-      /*float_ranges=*/{{0.0f, 1.0f}},
+          /*border mode*/ {}, roi);
      /*int_ranges=*/{{0, 255}, {-128, 127}},
      /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
      /*border mode*/ {}, roi);
 }
 TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
@@ -255,11 +262,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) {
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect_border_zero.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/"
                 "medium_sub_rect_keep_aspect_border_zero.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -273,11 +277,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) {
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(M_PI * 90.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("medium_sub_rect_keep_aspect_with_rotation.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/"
                 "medium_sub_rect_keep_aspect_with_rotation.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}},
          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -292,11 +293,9 @@ TEST(ImageToTensorCalculatorTest,
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(M_PI * 90.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath(
-          GetRgb("/mediapipe/calculators/"
+              "medium_sub_rect_keep_aspect_with_rotation_border_zero.png")),
                 "tensor/testdata/image_to_tensor/"
                 "medium_sub_rect_keep_aspect_with_rotation_border_zero.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true,
@@ -310,16 +309,12 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) {
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(M_PI * -45.0f / 180.0f);
-  RunTest(
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-      GetRgb("/mediapipe/calculators/"
+          GetRgb(GetFilePath("medium_sub_rect_with_rotation.png")),
-             "tensor/testdata/image_to_tensor/input.jpg"),
+          /*float_ranges=*/{{-1.0f, 1.0f}},
-      GetRgb(
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
-          "/mediapipe/calculators/"
+          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
-          "tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"),
+          BorderMode::kReplicate, roi);
      /*float_ranges=*/{{-1.0f, 1.0f}},
      /*int_ranges=*/{{0, 255}, {-128, 127}},
      /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
      BorderMode::kReplicate, roi);
 }
 TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
@@ -329,11 +324,8 @@ TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) {
  roi.set_width(0.5f);
  roi.set_height(0.5f);
  roi.set_rotation(M_PI * -45.0f / 180.0f);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("medium_sub_rect_with_rotation_border_zero.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/"
                 "medium_sub_rect_with_rotation_border_zero.png"),
          /*float_ranges=*/{{-1.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false,
@@ -347,10 +339,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRect) {
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("large_sub_rect.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/large_sub_rect.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}},
          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
@@ -364,15 +354,12 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) {
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(0);
-  RunTest(
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-      GetRgb("/mediapipe/calculators/"
+          GetRgb(GetFilePath("large_sub_rect_border_zero.png")),
-             "tensor/testdata/image_to_tensor/input.jpg"),
+          /*float_ranges=*/{{0.0f, 1.0f}},
-      GetRgb("/mediapipe/calculators/"
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
-             "tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"),
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
-      /*float_ranges=*/{{0.0f, 1.0f}},
+          BorderMode::kZero, roi);
      /*int_ranges=*/{{0, 255}, {-128, 127}},
      /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false,
      BorderMode::kZero, roi);
 }
 TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
@@ -382,15 +369,12 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) {
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(0);
-  RunTest(
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-      GetRgb("/mediapipe/calculators/"
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect.png")),
-             "tensor/testdata/image_to_tensor/input.jpg"),
+          /*float_ranges=*/{{0.0f, 1.0f}},
-      GetRgb("/mediapipe/calculators/"
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
-             "tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"),
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
-      /*float_ranges=*/{{0.0f, 1.0f}},
+          BorderMode::kReplicate, roi);
      /*int_ranges=*/{{0, 255}, {-128, 127}},
      /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
      BorderMode::kReplicate, roi);
 }
 TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
@@ -400,11 +384,8 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) {
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(0);
-  RunTest(GetRgb("/mediapipe/calculators/"
+  RunTest(GetRgb(GetFilePath("input.jpg")),
-                 "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect_border_zero.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/"
                 "large_sub_rect_keep_aspect_border_zero.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -418,11 +399,23 @@ TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) {
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(M_PI * -15.0f / 180.0f);
-  RunTest(GetRgba("/mediapipe/calculators/"
+  RunTest(GetRgba(GetFilePath("input.jpg")),
-                  "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("large_sub_rect_keep_aspect_with_rotation.png")),
-          GetRgb("/mediapipe/calculators/"
+          /*float_ranges=*/{{0.0f, 1.0f}},
-                 "tensor/testdata/image_to_tensor/"
+          /*int_ranges=*/{{0, 255}, {-128, 127}},
-                 "large_sub_rect_keep_aspect_with_rotation.png"),
+          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
          /*border_mode=*/{}, roi);
 }
 TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotationGray) {
  mediapipe::NormalizedRect roi;
  roi.set_x_center(0.5f);
  roi.set_y_center(0.5f);
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(M_PI * -15.0f / 180.0f);
  RunTest(GetGray(GetFilePath("input.jpg")),
          GetGray(GetFilePath("large_sub_rect_keep_aspect_with_rotation.png")),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -437,11 +430,26 @@ TEST(ImageToTensorCalculatorTest,
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(M_PI * -15.0f / 180.0f);
-  RunTest(GetRgba("/mediapipe/calculators/"
+  RunTest(GetRgba(GetFilePath("input.jpg")),
-                  "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath(
-          GetRgb("/mediapipe/calculators/"
+              "large_sub_rect_keep_aspect_with_rotation_border_zero.png")),
-                 "tensor/testdata/image_to_tensor/"
+          /*float_ranges=*/{{0.0f, 1.0f}},
-                 "large_sub_rect_keep_aspect_with_rotation_border_zero.png"),
+          /*int_ranges=*/{{0, 255}},
          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
          /*border_mode=*/BorderMode::kZero, roi);
 }
 TEST(ImageToTensorCalculatorTest,
     LargeSubRectKeepAspectWithRotationBorderZeroGray) {
  mediapipe::NormalizedRect roi;
  roi.set_x_center(0.5f);
  roi.set_y_center(0.5f);
  roi.set_width(1.5f);
  roi.set_height(1.1f);
  roi.set_rotation(M_PI * -15.0f / 180.0f);
  RunTest(GetGray(GetFilePath("input.jpg")),
          GetGray(GetFilePath(
              "large_sub_rect_keep_aspect_with_rotation_border_zero.png")),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}},
          /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -455,10 +463,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRange) {
  roi.set_width(1.0f);
  roi.set_height(1.0f);
  roi.set_rotation(0);
-  RunTest(GetRgba("/mediapipe/calculators/"
+  RunTest(GetRgba(GetFilePath("input.jpg")),
-                  "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("noop_except_range.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/noop_except_range.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -472,10 +478,8 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) {
  roi.set_width(1.0f);
  roi.set_height(1.0f);
  roi.set_rotation(0);
-  RunTest(GetRgba("/mediapipe/calculators/"
+  RunTest(GetRgba(GetFilePath("input.jpg")),
-                  "tensor/testdata/image_to_tensor/input.jpg"),
+          GetRgb(GetFilePath("noop_except_range.png")),
          GetRgb("/mediapipe/calculators/"
                 "tensor/testdata/image_to_tensor/noop_except_range.png"),
          /*float_ranges=*/{{0.0f, 1.0f}},
          /*int_ranges=*/{{0, 255}, {-128, 127}},
          /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true,
@@ -27,12 +27,6 @@ struct Size {
  int height;
 };
 // Pixel extrapolation method.
 // When converting image to tensor it may happen that tensor needs to read
 // pixels outside image boundaries. Border mode helps to specify how such pixels
 // will be calculated.
 enum class BorderMode { kZero, kReplicate };
 // Converts image to tensor.
 class ImageToTensorConverter {
 public:
@@ -42,13 +36,16 @@ class ImageToTensorConverter {
  // @image contains image to extract from.
  // @roi describes region of interest within the image to extract (absolute
  // values).
  // @output_dims dimensions of output tensor.
  // @range_min/max describes output tensor range image pixels should converted
  // to.
-  virtual absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
+  // @tensor_buffer_offset an inteter representing the offset of the tensor
-                                         const RotatedRect& roi,
+  // buffer the result should be written to.
-                                         const Size& output_dims,
+  // @output_tensor a tensor with pre-defined shape. The "Convert" is
-                                         float range_min, float range_max) = 0;
+  // responsible of populating the content into the output tensor.
  virtual absl::Status Convert(const mediapipe::Image& input,
                               const RotatedRect& roi, float range_min,
                               float range_max, int tensor_buffer_offset,
                               Tensor& output_tensor) = 0;
 };
 }  // namespace mediapipe
@@ -264,57 +264,58 @@ class GlProcessor : public ImageToTensorConverter {
    });
  }
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
-                                 const RotatedRect& roi,
+                       float range_min, float range_max,
-                                 const Size& output_dims, float range_min,
+                       int tensor_buffer_offset,
-                                 float range_max) override {
+                       Tensor& output_tensor) override {
    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
-        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128 &&
        input.format() != mediapipe::GpuBufferFormat::kRGB24) {
      return InvalidArgumentError(absl::StrCat(
-          "Only 4-channel texture input formats are supported, passed format: ",
+          "Unsupported format: ", static_cast<uint32_t>(input.format())));
          static_cast<uint32_t>(input.format())));
    }
    const auto& output_shape = output_tensor.shape();
    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
-    constexpr int kNumChannels = 3;
+    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
-    Tensor tensor(Tensor::ElementType::kFloat32,
+        [this, &output_tensor, &input, &roi, &output_shape, range_min,
-                  {1, output_dims.height, output_dims.width, kNumChannels});
+         range_max, tensor_buffer_offset]() -> absl::Status {
          const int input_num_channels = input.channels();
          auto source_texture = gl_helper_.CreateSourceTexture(input);
          tflite::gpu::gl::GlTexture input_texture(
              GL_TEXTURE_2D, source_texture.name(),
              input_num_channels == 4 ? GL_RGBA : GL_RGB,
              source_texture.width() * source_texture.height() *
                  input_num_channels * sizeof(uint8_t),
              /*layer=*/0,
              /*owned=*/false);
-    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext([this, &tensor, &input, &roi,
+          constexpr float kInputImageRangeMin = 0.0f;
-                                                  &output_dims, range_min,
+          constexpr float kInputImageRangeMax = 1.0f;
-                                                  range_max]() -> absl::Status {
+          ASSIGN_OR_RETURN(auto transform,
-      constexpr int kRgbaNumChannels = 4;
+                           GetValueRangeTransformation(kInputImageRangeMin,
-      auto source_texture = gl_helper_.CreateSourceTexture(input);
+                                                       kInputImageRangeMax,
-      tflite::gpu::gl::GlTexture input_texture(
+                                                       range_min, range_max));
          GL_TEXTURE_2D, source_texture.name(), GL_RGBA,
          source_texture.width() * source_texture.height() * kRgbaNumChannels *
              sizeof(uint8_t),
          /*layer=*/0,
          /*owned=*/false);
-      constexpr float kInputImageRangeMin = 0.0f;
+          const int output_size = output_tensor.bytes() / output_shape.dims[0];
-      constexpr float kInputImageRangeMax = 1.0f;
+          auto buffer_view = output_tensor.GetOpenGlBufferWriteView();
-      ASSIGN_OR_RETURN(
+          tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER,
-          auto transform,
+                                           buffer_view.name(), output_size,
-          GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
+                                           /*offset=*/tensor_buffer_offset,
-                                      range_min, range_max));
+                                           /*has_ownership=*/false);
          MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer(
              input_texture,
              tflite::gpu::HW(source_texture.height(), source_texture.width()),
              roi,
              /*flip_horizontaly=*/false, transform.scale, transform.offset,
              tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
              command_queue_.get(), &output));
-      auto buffer_view = tensor.GetOpenGlBufferWriteView();
+          return absl::OkStatus();
-      tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER,
+        }));
                                       buffer_view.name(), tensor.bytes(),
                                       /*offset=*/0,
                                       /*has_ownership=*/false);
      MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer(
          input_texture,
          tflite::gpu::HW(source_texture.height(), source_texture.width()), roi,
          /*flip_horizontaly=*/false, transform.scale, transform.offset,
          tflite::gpu::HW(output_dims.height, output_dims.width),
          command_queue_.get(), &output));
-      return absl::OkStatus();
+    return absl::OkStatus();
    }));
    return tensor;
  }
  ~GlProcessor() override {
@@ -326,6 +327,16 @@ class GlProcessor : public ImageToTensorConverter {
  }
 private:
  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
    RET_CHECK_EQ(output_shape.dims.size(), 4)
        << "Wrong output dims size: " << output_shape.dims.size();
    RET_CHECK_GE(output_shape.dims[0], 1)
        << "The batch dimension needs to be greater or equal to 1.";
    RET_CHECK_EQ(output_shape.dims[3], 3)
        << "Wrong output channel: " << output_shape.dims[3];
    return absl::OkStatus();
  }
  std::unique_ptr<tflite::gpu::gl::CommandQueue> command_queue_;
  std::unique_ptr<SubRectExtractorGl> extractor_;
  mediapipe::GlCalculatorHelper gl_helper_;
@@ -68,8 +68,8 @@ class GlProcessor : public ImageToTensorConverter {
      constexpr GLchar kExtractSubRectVertexShader[] = R"(
            in vec4 position;
-            in mediump vec4 texture_coordinate;
+            in highp vec4 texture_coordinate;
-            out mediump vec2 sample_coordinate;
+            out highp vec2 sample_coordinate;
            uniform mat4 transform_matrix;
            void main() {
@@ -86,7 +86,7 @@ class GlProcessor : public ImageToTensorConverter {
          )";
      constexpr GLchar kExtractSubRectFragBody[] = R"(
-            DEFAULT_PRECISION(mediump, float)
+            DEFAULT_PRECISION(highp, float)
            // Provided by kExtractSubRectVertexShader.
            in vec2 sample_coordinate;
@@ -168,26 +168,26 @@ class GlProcessor : public ImageToTensorConverter {
    });
  }
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
-                                 const RotatedRect& roi,
+                       float range_min, float range_max,
-                                 const Size& output_dims, float range_min,
+                       int tensor_buffer_offset,
-                                 float range_max) override {
+                       Tensor& output_tensor) override {
    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
-        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
+        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128 &&
        input.format() != mediapipe::GpuBufferFormat::kRGB24) {
      return InvalidArgumentError(absl::StrCat(
-          "Only 4-channel texture input formats are supported, passed format: ",
+          "Unsupported format: ", static_cast<uint32_t>(input.format())));
          static_cast<uint32_t>(input.format())));
    }
    // TODO: support tensor_buffer_offset > 0 scenario.
    RET_CHECK_EQ(tensor_buffer_offset, 0)
        << "The non-zero tensor_buffer_offset input is not supported yet.";
    const auto& output_shape = output_tensor.shape();
    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
-    constexpr int kNumChannels = 3;
+    MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext(
-    Tensor tensor(
+        [this, &output_tensor, &input, &roi, &output_shape, range_min,
-        Tensor::ElementType::kFloat32,
+         range_max]() -> absl::Status {
        Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels});
    MP_RETURN_IF_ERROR(
        gl_helper_.RunInGlContext([this, &tensor, &input, &roi, &output_dims,
                                   range_min, range_max]() -> absl::Status {
          auto input_texture = gl_helper_.CreateSourceTexture(input);
          constexpr float kInputImageRangeMin = 0.0f;
@@ -196,27 +196,29 @@ class GlProcessor : public ImageToTensorConverter {
                           GetValueRangeTransformation(kInputImageRangeMin,
                                                       kInputImageRangeMax,
                                                       range_min, range_max));
-          auto tensor_view = tensor.GetOpenGlTexture2dWriteView();
+          auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
          MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
                                            /*flip_horizontaly=*/false,
                                            transform.scale, transform.offset,
-                                            output_dims, &tensor_view));
+                                            output_shape, &tensor_view));
          return absl::OkStatus();
        }));
-    return tensor;
+    return absl::OkStatus();
  }
  absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
                              const RotatedRect& sub_rect,
                              bool flip_horizontaly, float alpha, float beta,
-                              const Size& output_dims,
+                              const Tensor::Shape& output_shape,
                              Tensor::OpenGlTexture2dView* output) {
    const int output_height = output_shape.dims[1];
    const int output_width = output_shape.dims[2];
    std::array<float, 16> transform_mat;
    glDisable(GL_DEPTH_TEST);
    glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
-    glViewport(0, 0, output_dims.width, output_dims.height);
+    glViewport(0, 0, output_width, output_height);
    glActiveTexture(GL_TEXTURE0);
    glBindTexture(GL_TEXTURE_2D, output->name());
@@ -316,6 +318,17 @@ class GlProcessor : public ImageToTensorConverter {
  }
 private:
  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
    RET_CHECK_EQ(output_shape.dims.size(), 4)
        << "Wrong output dims size: " << output_shape.dims.size();
    RET_CHECK_EQ(output_shape.dims[0], 1)
        << "Handling batch dimension not equal to 1 is not implemented in this "
           "converter.";
    RET_CHECK_EQ(output_shape.dims[3], 3)
        << "Wrong output channel: " << output_shape.dims[3];
    return absl::OkStatus();
  }
  mediapipe::GlCalculatorHelper gl_helper_;
  bool use_custom_zero_border_ = false;
  BorderMode border_mode_ = BorderMode::kReplicate;
@@ -36,6 +36,10 @@
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/common/types.h"
 #if MEDIAPIPE_METAL_ENABLED
 #include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
 namespace mediapipe {
 namespace {
@@ -262,7 +266,6 @@ class SubRectExtractorMetal {
    RET_CHECK(pipeline_state != nil);
    std::string output_type_def;
    MTLPixelFormat pixel_format;
    switch (output_format) {
      case OutputFormat::kF16C4:
        output_type_def = R"(
@@ -348,10 +351,10 @@ class MetalProcessor : public ImageToTensorConverter {
    return absl::OkStatus();
  }
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
-                                 const RotatedRect& roi,
+                       float range_min, float range_max,
-                                 const Size& output_dims, float range_min,
+                       int tensor_buffer_offset,
-                                 float range_max) override {
+                       Tensor& output_tensor) override {
    if (input.format() != mediapipe::GpuBufferFormat::kBGRA32 &&
        input.format() != mediapipe::GpuBufferFormat::kRGBAHalf64 &&
        input.format() != mediapipe::GpuBufferFormat::kRGBAFloat128) {
@@ -359,16 +362,15 @@ class MetalProcessor : public ImageToTensorConverter {
          "Only 4-channel texture input formats are supported, passed format: ",
          static_cast<uint32_t>(input.format())));
    }
    RET_CHECK_EQ(tensor_buffer_offset, 0)
        << "The non-zero tensor_buffer_offset input is not supported yet.";
    const auto& output_shape = output_tensor.shape();
    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
    @autoreleasepool {
      id<MTLTexture> texture =
          [metal_helper_ metalTextureWithGpuBuffer:input.GetGpuBuffer()];
      constexpr int kNumChannels = 4;
      Tensor tensor(Tensor::ElementType::kFloat32,
                    Tensor::Shape{1, output_dims.height, output_dims.width,
                                  kNumChannels});
      constexpr float kInputImageRangeMin = 0.0f;
      constexpr float kInputImageRangeMax = 1.0f;
      ASSIGN_OR_RETURN(
@@ -377,18 +379,30 @@ class MetalProcessor : public ImageToTensorConverter {
                                      range_min, range_max));
      id<MTLCommandBuffer> command_buffer = [metal_helper_ commandBuffer];
-      const auto& buffer_view = tensor.GetMtlBufferWriteView(command_buffer);
+      const auto& buffer_view =
          MtlBufferView::GetWriteView(output_tensor, command_buffer);
      MP_RETURN_IF_ERROR(extractor_->Execute(
          texture, roi,
          /*flip_horizontaly=*/false, transform.scale, transform.offset,
-          tflite::gpu::HW(output_dims.height, output_dims.width),
+          tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
          command_buffer, buffer_view.buffer()));
      [command_buffer commit];
-      return tensor;
+      return absl::OkStatus();
    }
  }
 private:
  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
    RET_CHECK_EQ(output_shape.dims.size(), 4)
        << "Wrong output dims size: " << output_shape.dims.size();
    RET_CHECK_EQ(output_shape.dims[0], 1)
        << "Handling batch dimension not equal to 1 is not implemented in this "
           "converter.";
    RET_CHECK_EQ(output_shape.dims[3], 4)
        << "Wrong output channel: " << output_shape.dims[3];
    return absl::OkStatus();
  }
  MPPMetalHelper* metal_helper_ = nil;
  std::unique_ptr<SubRectExtractorMetal> extractor_;
 };
@@ -48,47 +48,77 @@ class OpenCvProcessor : public ImageToTensorConverter {
    switch (tensor_type_) {
      case Tensor::ElementType::kInt8:
        mat_type_ = CV_8SC3;
        mat_gray_type_ = CV_8SC1;
        break;
      case Tensor::ElementType::kFloat32:
        mat_type_ = CV_32FC3;
        mat_gray_type_ = CV_32FC1;
        break;
      case Tensor::ElementType::kUInt8:
        mat_type_ = CV_8UC3;
        mat_gray_type_ = CV_8UC1;
        break;
      default:
        mat_type_ = -1;
        mat_gray_type_ = -1;
    }
  }
-  absl::StatusOr<Tensor> Convert(const mediapipe::Image& input,
+  absl::Status Convert(const mediapipe::Image& input, const RotatedRect& roi,
-                                 const RotatedRect& roi,
+                       float range_min, float range_max,
-                                 const Size& output_dims, float range_min,
+                       int tensor_buffer_offset,
-                                 float range_max) override {
+                       Tensor& output_tensor) override {
-    if (input.image_format() != mediapipe::ImageFormat::SRGB &&
+    const bool is_supported_format =
-        input.image_format() != mediapipe::ImageFormat::SRGBA) {
+        input.image_format() == mediapipe::ImageFormat::SRGB ||
-      return InvalidArgumentError(
+        input.image_format() == mediapipe::ImageFormat::SRGBA ||
-          absl::StrCat("Only RGBA/RGB formats are supported, passed format: ",
+        input.image_format() == mediapipe::ImageFormat::GRAY8;
-                       static_cast<uint32_t>(input.image_format())));
+    if (!is_supported_format) {
      return InvalidArgumentError(absl::StrCat(
          "Unsupported format: ", static_cast<uint32_t>(input.image_format())));
    }
    auto src = mediapipe::formats::MatView(&input);
-    constexpr int kNumChannels = 3;
+    RET_CHECK_GE(tensor_buffer_offset, 0)
-    Tensor tensor(tensor_type_, Tensor::Shape{1, output_dims.height,
+        << "The input tensor_buffer_offset needs to be non-negative.";
-                                              output_dims.width, kNumChannels});
+    const auto& output_shape = output_tensor.shape();
-    auto buffer_view = tensor.GetCpuWriteView();
+    MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
    const int output_height = output_shape.dims[1];
    const int output_width = output_shape.dims[2];
    const int output_channels = output_shape.dims[3];
    const int num_elements_per_img =
        output_height * output_width * output_channels;
    auto buffer_view = output_tensor.GetCpuWriteView();
    cv::Mat dst;
    const int dst_data_type = output_channels == 1 ? mat_gray_type_ : mat_type_;
    switch (tensor_type_) {
      case Tensor::ElementType::kInt8:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+        RET_CHECK_GE(output_shape.num_elements(),
-                      buffer_view.buffer<int8>());
+                     tensor_buffer_offset / sizeof(int8) + num_elements_per_img)
            << "The buffer offset + the input image size is larger than the "
               "allocated tensor buffer.";
        dst = cv::Mat(
            output_height, output_width, dst_data_type,
            buffer_view.buffer<int8>() + tensor_buffer_offset / sizeof(int8));
        break;
      case Tensor::ElementType::kFloat32:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+        RET_CHECK_GE(
-                      buffer_view.buffer<float>());
+            output_shape.num_elements(),
            tensor_buffer_offset / sizeof(float) + num_elements_per_img)
            << "The buffer offset + the input image size is larger than the "
               "allocated tensor buffer.";
        dst = cv::Mat(
            output_height, output_width, dst_data_type,
            buffer_view.buffer<float>() + tensor_buffer_offset / sizeof(float));
        break;
      case Tensor::ElementType::kUInt8:
-        dst = cv::Mat(output_dims.height, output_dims.width, mat_type_,
+        RET_CHECK_GE(
-                      buffer_view.buffer<uint8>());
+            output_shape.num_elements(),
            tensor_buffer_offset / sizeof(uint8) + num_elements_per_img)
            << "The buffer offset + the input image size is larger than the "
               "allocated tensor buffer.";
        dst = cv::Mat(
            output_height, output_width, dst_data_type,
            buffer_view.buffer<uint8>() + tensor_buffer_offset / sizeof(uint8));
        break;
      default:
        return InvalidArgumentError(
@@ -101,8 +131,8 @@ class OpenCvProcessor : public ImageToTensorConverter {
    cv::Mat src_points;
    cv::boxPoints(rotated_rect, src_points);
-    const float dst_width = output_dims.width;
+    const float dst_width = output_width;
-    const float dst_height = output_dims.height;
+    const float dst_height = output_height;
    /* clang-format off */
    float dst_corners[8] = {0.0f,      dst_height,
                            0.0f,      0.0f,
@@ -110,6 +140,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
                            dst_width, dst_height};
    /* clang-format on */
    auto src = mediapipe::formats::MatView(&input);
    cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
    cv::Mat projection_matrix =
        cv::getPerspectiveTransform(src_points, dst_points);
@@ -119,7 +150,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
                        /*flags=*/cv::INTER_LINEAR,
                        /*borderMode=*/border_mode_);
-    if (transformed.channels() > kNumChannels) {
+    if (transformed.channels() > output_channels) {
      cv::Mat proper_channels_mat;
      cv::cvtColor(transformed, proper_channels_mat, cv::COLOR_RGBA2RGB);
      transformed = proper_channels_mat;
@@ -131,14 +162,26 @@ class OpenCvProcessor : public ImageToTensorConverter {
        auto transform,
        GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                    range_min, range_max));
-    transformed.convertTo(dst, mat_type_, transform.scale, transform.offset);
+    transformed.convertTo(dst, dst_data_type, transform.scale,
-    return tensor;
+                          transform.offset);
    return absl::OkStatus();
  }
 private:
  absl::Status ValidateTensorShape(const Tensor::Shape& output_shape) {
    RET_CHECK_EQ(output_shape.dims.size(), 4)
        << "Wrong output dims size: " << output_shape.dims.size();
    RET_CHECK_GE(output_shape.dims[0], 1)
        << "The batch dimension needs to be equal or larger than 1.";
    RET_CHECK(output_shape.dims[3] == 3 || output_shape.dims[3] == 1)
        << "Wrong output channel: " << output_shape.dims[3];
    return absl::OkStatus();
  }
  enum cv::BorderTypes border_mode_;
  Tensor::ElementType tensor_type_;
  int mat_type_;
  int mat_gray_type_;
 };
 }  // namespace
@@ -16,7 +16,9 @@
 #include <array>
 #include "absl/status/status.h"
 #include "absl/types/optional.h"
 #include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/statusor.h"
@@ -214,4 +216,75 @@ void GetTransposedRotatedSubRectToRectTransformMatrix(
  matrix[15] = 1.0f;
 }
 BorderMode GetBorderMode(
    const mediapipe::ImageToTensorCalculatorOptions::BorderMode& mode) {
  switch (mode) {
    case mediapipe::
        ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED:
      return BorderMode::kReplicate;
    case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO:
      return BorderMode::kZero;
    case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE:
      return BorderMode::kReplicate;
  }
 }
 Tensor::ElementType GetOutputTensorType(bool uses_gpu,
                                        const OutputTensorParams& params) {
  if (!uses_gpu) {
    if (params.is_float_output) {
      return Tensor::ElementType::kFloat32;
    }
    if (params.range_min < 0) {
      return Tensor::ElementType::kInt8;
    } else {
      return Tensor::ElementType::kUInt8;
    }
  }
  // Always use float32 when GPU is enabled.
  return Tensor::ElementType::kFloat32;
 }
 int GetNumOutputChannels(const mediapipe::Image& image) {
 #if !MEDIAPIPE_DISABLE_GPU
 #if MEDIAPIPE_METAL_ENABLED
  if (image.UsesGpu()) {
    return 4;
  }
 #endif  // MEDIAPIPE_METAL_ENABLED
 #endif  // !MEDIAPIPE_DISABLE_GPU
  // TODO: Add a unittest here to test the behavior on GPU, i.e.
  // failure.
  // Only output channel == 1 when running on CPU and the input image channel
  // is 1. Ideally, we want to also support GPU for output channel == 1. But
  // setting this on the safer side to prevent unintentional failure.
  if (!image.UsesGpu() && image.channels() == 1) {
    return 1;
  }
  return 3;
 }
 absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
    const api2::Packet<api2::OneOf<Image, mediapipe::ImageFrame>>&
        image_packet) {
  return image_packet.Visit(
      [&image_packet](const mediapipe::Image&) {
        return SharedPtrWithPacket<mediapipe::Image>(image_packet);
      },
      [&image_packet](const mediapipe::ImageFrame&) {
        return std::make_shared<const mediapipe::Image>(
            std::const_pointer_cast<mediapipe::ImageFrame>(
                SharedPtrWithPacket<mediapipe::ImageFrame>(image_packet)));
      });
 }
 #if !MEDIAPIPE_DISABLE_GPU
 absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
    const api2::Packet<mediapipe::GpuBuffer>& image_gpu_packet) {
  // A shallow copy is okay since the resulting 'image' object is local in
  // Process(), and thus never outlives 'input'.
  return std::make_shared<const mediapipe::Image>(image_gpu_packet.Get());
 }
 #endif  // !MEDIAPIPE_DISABLE_GPU
 }  // namespace mediapipe
@@ -18,8 +18,18 @@
 #include <array>
 #include "absl/types/optional.h"
 #include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/packet.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/statusor.h"
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_origin.pb.h"
 namespace mediapipe {
@@ -31,6 +41,24 @@ struct RotatedRect {
  float rotation;
 };
 // Pixel extrapolation method.
 // When converting image to tensor it may happen that tensor needs to read
 // pixels outside image boundaries. Border mode helps to specify how such pixels
 // will be calculated.
 // TODO: Consider moving this to a separate border_mode.h file.
 enum class BorderMode { kZero, kReplicate };
 // Struct that host commonly accessed parameters used in the
 // ImageTo[Batch]TensorCalculator.
 struct OutputTensorParams {
  int output_height;
  int output_width;
  int output_batch;
  bool is_float_output;
  float range_min;
  float range_max;
 };
 // Generates a new ROI or converts it from normalized rect.
 RotatedRect GetRoi(int input_width, int input_height,
                   absl::optional<mediapipe::NormalizedRect> norm_rect);
@@ -95,6 +123,103 @@ void GetTransposedRotatedSubRectToRectTransformMatrix(
    const RotatedRect& sub_rect, int rect_width, int rect_height,
    bool flip_horizontaly, std::array<float, 16>* matrix);
 // Validates the output dimensions set in the option proto. The input option
 // proto is expected to have to following fields:
 //  output_tensor_float_range, output_tensor_int_range, output_tensor_uint_range
 //  output_tensor_width, output_tensor_height.
 // See ImageToTensorCalculatorOptions for the description of each field.
 template <typename T>
 absl::Status ValidateOptionOutputDims(const T& options) {
  RET_CHECK(options.has_output_tensor_float_range() ||
            options.has_output_tensor_int_range() ||
            options.has_output_tensor_uint_range())
      << "Output tensor range is required.";
  if (options.has_output_tensor_float_range()) {
    RET_CHECK_LT(options.output_tensor_float_range().min(),
                 options.output_tensor_float_range().max())
        << "Valid output float tensor range is required.";
  }
  if (options.has_output_tensor_uint_range()) {
    RET_CHECK_LT(options.output_tensor_uint_range().min(),
                 options.output_tensor_uint_range().max())
        << "Valid output uint tensor range is required.";
    RET_CHECK_GE(options.output_tensor_uint_range().min(), 0)
        << "The minimum of the output uint tensor range must be "
           "non-negative.";
    RET_CHECK_LE(options.output_tensor_uint_range().max(), 255)
        << "The maximum of the output uint tensor range must be less than or "
           "equal to 255.";
  }
  if (options.has_output_tensor_int_range()) {
    RET_CHECK_LT(options.output_tensor_int_range().min(),
                 options.output_tensor_int_range().max())
        << "Valid output int tensor range is required.";
    RET_CHECK_GE(options.output_tensor_int_range().min(), -128)
        << "The minimum of the output int tensor range must be greater than "
           "or equal to -128.";
    RET_CHECK_LE(options.output_tensor_int_range().max(), 127)
        << "The maximum of the output int tensor range must be less than or "
           "equal to 127.";
  }
  RET_CHECK_GT(options.output_tensor_width(), 0)
      << "Valid output tensor width is required.";
  RET_CHECK_GT(options.output_tensor_height(), 0)
      << "Valid output tensor height is required.";
  return absl::OkStatus();
 }
 template <typename T>
 OutputTensorParams GetOutputTensorParams(const T& options) {
  OutputTensorParams params;
  if (options.has_output_tensor_uint_range()) {
    params.range_min =
        static_cast<float>(options.output_tensor_uint_range().min());
    params.range_max =
        static_cast<float>(options.output_tensor_uint_range().max());
  } else if (options.has_output_tensor_int_range()) {
    params.range_min =
        static_cast<float>(options.output_tensor_int_range().min());
    params.range_max =
        static_cast<float>(options.output_tensor_int_range().max());
  } else {
    params.range_min = options.output_tensor_float_range().min();
    params.range_max = options.output_tensor_float_range().max();
  }
  params.output_width = options.output_tensor_width();
  params.output_height = options.output_tensor_height();
  params.is_float_output = options.has_output_tensor_float_range();
  params.output_batch = 1;
  return params;
 }
 // Returns whether the GPU input format starts at the bottom.
 template <typename T>
 bool DoesGpuInputStartAtBottom(const T& options) {
  return options.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
 }
 // Converts the BorderMode proto into struct.
 BorderMode GetBorderMode(
    const mediapipe::ImageToTensorCalculatorOptions::BorderMode& mode);
 // Gets the output tensor type.
 Tensor::ElementType GetOutputTensorType(bool uses_gpu,
                                        const OutputTensorParams& params);
 // Gets the number of output channels from the input Image format.
 int GetNumOutputChannels(const mediapipe::Image& image);
 // Converts the packet that hosts different format (Image, ImageFrame,
 // GpuBuffer) into the mediapipe::Image format.
 absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
    const api2::Packet<api2::OneOf<Image, mediapipe::ImageFrame>>&
        image_packet);
 #if !MEDIAPIPE_DISABLE_GPU
 absl::StatusOr<std::shared_ptr<const mediapipe::Image>> GetInputImage(
    const api2::Packet<mediapipe::GpuBuffer>& image_gpu_packet);
 #endif  // !MEDIAPIPE_DISABLE_GPU
 }  // namespace mediapipe
 #endif  // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_
@@ -16,6 +16,8 @@
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_macros.h"
 #include "mediapipe/framework/port/status_matchers.h"
 namespace mediapipe {
@@ -23,6 +25,7 @@ namespace {
 using ::testing::ElementsAre;
 using ::testing::ElementsAreArray;
 using ::testing::HasSubstr;
 testing::Matcher<RotatedRect> EqRotatedRect(float width, float height,
                                            float center_x, float center_y,
@@ -157,5 +160,95 @@ TEST(GetValueRangeTransformation, FloatToPixel) {
              EqValueTransformation(/*scale=*/255.0f, /*offset=*/0.0f));
 }
 constexpr char kValidFloatProto[] = R"(
  output_tensor_float_range { min: 0.0 max: 1.0 }
  output_tensor_width: 100
  output_tensor_height: 200
 )";
 constexpr char kValidIntProto[] = R"(
  output_tensor_float_range { min: 0 max: 255 }
  output_tensor_width: 100
  output_tensor_height: 200
 )";
 TEST(ValidateOptionOutputDims, ImageToTensorCalcOptions) {
  const auto float_options =
      mediapipe::ParseTextProtoOrDie<mediapipe::ImageToTensorCalculatorOptions>(
          kValidFloatProto);
  MP_EXPECT_OK(ValidateOptionOutputDims(float_options));
 }
 TEST(ValidateOptionOutputDims, EmptyProto) {
  mediapipe::ImageToTensorCalculatorOptions options;
  // No output tensor range set.
  EXPECT_THAT(ValidateOptionOutputDims(options),
              StatusIs(absl::StatusCode::kInternal,
                       HasSubstr("Output tensor range is required")));
  // Invalid output float tensor range.
  options.mutable_output_tensor_float_range()->set_min(1.0);
  options.mutable_output_tensor_float_range()->set_max(0.0);
  EXPECT_THAT(
      ValidateOptionOutputDims(options),
      StatusIs(absl::StatusCode::kInternal,
               HasSubstr("Valid output float tensor range is required")));
  // Output width/height is not set.
  options.mutable_output_tensor_float_range()->set_min(0.0);
  options.mutable_output_tensor_float_range()->set_max(1.0);
  EXPECT_THAT(ValidateOptionOutputDims(options),
              StatusIs(absl::StatusCode::kInternal,
                       HasSubstr("Valid output tensor width is required")));
 }
 TEST(GetOutputTensorParams, ImageToTensorCalcOptionsSetValues) {
  // Test int range with ImageToTensorCalculatorOptions.
  const auto int_options =
      mediapipe::ParseTextProtoOrDie<mediapipe::ImageToTensorCalculatorOptions>(
          kValidIntProto);
  const auto params2 = GetOutputTensorParams(int_options);
  EXPECT_EQ(params2.range_min, 0.0f);
  EXPECT_EQ(params2.range_max, 255.0f);
  EXPECT_EQ(params2.output_batch, 1);
  EXPECT_EQ(params2.output_width, 100);
  EXPECT_EQ(params2.output_height, 200);
 }
 TEST(GetBorderMode, GetBorderMode) {
  // Default to REPLICATE.
  auto border_mode =
      mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED;
  EXPECT_EQ(BorderMode::kReplicate, GetBorderMode(border_mode));
  // Set to ZERO.
  border_mode =
      mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO;
  EXPECT_EQ(BorderMode::kZero, GetBorderMode(border_mode));
 }
 TEST(GetOutputTensorType, GetOutputTensorType) {
  OutputTensorParams params;
  // Return float32 when GPU is enabled.
  EXPECT_EQ(Tensor::ElementType::kFloat32,
            GetOutputTensorType(/*uses_gpu=*/true, params));
  // Return float32 when is_float_output is set to true.
  params.is_float_output = true;
  EXPECT_EQ(Tensor::ElementType::kFloat32,
            GetOutputTensorType(/*uses_gpu=*/false, params));
  // Return int8 when range_min is negative.
  params.is_float_output = false;
  params.range_min = -255.0f;
  EXPECT_EQ(Tensor::ElementType::kInt8,
            GetOutputTensorType(/*uses_gpu=*/false, params));
  // Return 8int8 when range_min is non-negative.
  params.range_min = 0.0f;
  EXPECT_EQ(Tensor::ElementType::kUInt8,
            GetOutputTensorType(/*uses_gpu=*/false, params));
 }
 }  // namespace
 }  // namespace mediapipe
@@ -17,6 +17,7 @@ syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/calculator_options.proto";
 option java_package = "com.google.mediapipe.calculator.proto";
 option java_outer_classname = "InferenceCalculatorProto";
@@ -72,7 +72,7 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
  RET_CHECK(!input_tensors.empty());
  ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
-                   inference_runner_->Run(input_tensors));
+                   inference_runner_->Run(cc, input_tensors));
  kOutTensors(cc).Send(std::move(output_tensors));
  return absl::OkStatus();
 }
@@ -151,6 +151,8 @@ InferenceCalculatorCpuImpl::MaybeCreateDelegate(CalculatorContext* cc) {
    auto xnnpack_opts = TfLiteXNNPackDelegateOptionsDefault();
    xnnpack_opts.num_threads =
        GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
    // TODO Remove once XNNPACK is enabled by default.
    xnnpack_opts.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
    return TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
                             &TfLiteXNNPackDelegateDelete);
  }
@@ -20,12 +20,15 @@
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
 #include "mediapipe/calculators/tensor/inference_calculator.pb.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
 #define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
 namespace mediapipe {
 namespace api2 {
@@ -152,6 +155,10 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
  const auto& input_indices = interpreter_->inputs();
  for (int i = 0; i < input_indices.size(); ++i) {
    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
        "Input tensor at index [%d] doesn't specify dimensions.",
        input_indices[i]);
    gpu_buffers_in_.emplace_back(absl::make_unique<Tensor>(
        Tensor::ElementType::kFloat32,
        Tensor::Shape{std::vector<int>{
@@ -169,6 +176,9 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
  // Create and bind output buffers.
  for (int i = 0; i < output_size_; ++i) {
    const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
        "Output tensor at index [%d] doesn't specify dimensions.",
        output_indices[i]);
    gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
        Tensor::ElementType::kFloat32,
        Tensor::Shape{std::vector<int>{
@@ -191,7 +201,7 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
    CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
    std::vector<Tensor>& output_tensors) {
  return gpu_helper_.RunInGlContext(
-      [this, &input_tensors, &output_tensors]() -> absl::Status {
+      [this, cc, &input_tensors, &output_tensors]() -> absl::Status {
        // Explicitly copy input.
        for (int i = 0; i < input_tensors.size(); ++i) {
          glBindBuffer(GL_COPY_READ_BUFFER,
@@ -203,7 +213,10 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
        }
        // Run inference.
-        RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+        {
          MEDIAPIPE_PROFILING(GPU_TASK_INVOKE, cc);
          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
        }
        output_tensors.reserve(output_size_);
        for (int i = 0; i < output_size_; ++i) {
@@ -25,12 +25,14 @@
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/util/tflite/tflite_gpu_runner.h"
-#if defined(MEDIAPIPE_ANDROID)
+#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/filesystem.h"
 #include "mediapipe/util/android/file/base/helpers.h"
-#endif  // MEDIAPIPE_ANDROID
+#endif  // defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 #define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
 namespace mediapipe {
 namespace api2 {
@@ -83,7 +85,7 @@ class InferenceCalculatorGlAdvancedImpl
        const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
    absl::StatusOr<std::vector<Tensor>> Process(
-        const std::vector<Tensor>& input_tensors);
+        CalculatorContext* cc, const std::vector<Tensor>& input_tensors);
    absl::Status Close();
@@ -121,11 +123,11 @@ absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init(
 absl::StatusOr<std::vector<Tensor>>
 InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
-    const std::vector<Tensor>& input_tensors) {
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
  std::vector<Tensor> output_tensors;
  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-      [this, &input_tensors, &output_tensors]() -> absl::Status {
+      [this, cc, &input_tensors, &output_tensors]() -> absl::Status {
        for (int i = 0; i < input_tensors.size(); ++i) {
          MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
              input_tensors[i].GetOpenGlBufferReadView().name(), i));
@@ -138,7 +140,10 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
              output_tensors.back().GetOpenGlBufferWriteView().name(), i));
        }
        // Run inference.
-        return tflite_gpu_runner_->Invoke();
+        {
          MEDIAPIPE_PROFILING(GPU_TASK_INVOKE, cc);
          return tflite_gpu_runner_->Invoke();
        }
      }));
  return output_tensors;
@@ -226,19 +231,26 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
  return tflite_gpu_runner_->Build();
 }
-#if defined(MEDIAPIPE_ANDROID)
+#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
    const mediapipe::InferenceCalculatorOptions& options,
    const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
        gpu_delegate_options) {
-  use_kernel_caching_ = gpu_delegate_options.has_cached_kernel_path();
+  // The kernel cache needs a unique filename based on either model_path or the
  // model token, to prevent the cache from being overwritten if the graph has
  // more than one model.
  use_kernel_caching_ =
      gpu_delegate_options.has_cached_kernel_path() &&
      (options.has_model_path() || gpu_delegate_options.has_model_token());
  use_serialized_model_ = gpu_delegate_options.has_serialized_model_dir() &&
                          gpu_delegate_options.has_model_token();
  if (use_kernel_caching_) {
-    cached_kernel_filename_ = gpu_delegate_options.cached_kernel_path() +
+    std::string basename = options.has_model_path()
-                              mediapipe::File::Basename(options.model_path()) +
+                               ? mediapipe::File::Basename(options.model_path())
-                              ".ker";
+                               : gpu_delegate_options.model_token();
    cached_kernel_filename_ = mediapipe::file::JoinPath(
        gpu_delegate_options.cached_kernel_path(), basename + ".ker");
  }
  if (use_serialized_model_) {
    serialized_model_path_ =
@@ -253,9 +265,9 @@ InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
  if (use_kernel_caching_) {
    // Save kernel file.
-    auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
+    ASSIGN_OR_RETURN(std::vector<uint8_t> kernel_cache,
-        gpu_runner->GetSerializedBinaryCache());
+                     gpu_runner->GetSerializedBinaryCache());
-    std::string cache_str(kernel_cache->begin(), kernel_cache->end());
+    std::string cache_str(kernel_cache.begin(), kernel_cache.end());
    MP_RETURN_IF_ERROR(
        mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
  }
@@ -313,7 +325,7 @@ InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
  return absl::OkStatus();
 }
-#endif  // MEDIAPIPE_ANDROID
+#endif  // defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
 absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
    CalculatorContract* cc) {
@@ -354,7 +366,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
  auto output_tensors = absl::make_unique<std::vector<Tensor>>();
  ASSIGN_OR_RETURN(*output_tensors,
-                   gpu_inference_runner_->Process(input_tensors));
+                   gpu_inference_runner_->Process(cc, input_tensors));
  kOutTensors(cc).Send(std::move(output_tensors));
  return absl::OkStatus();
@@ -22,7 +22,10 @@
 #include <vector>
 #include "absl/memory/memory.h"
 #include "absl/strings/str_format.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #include "mediapipe/gpu/gpu_buffer.h"
@@ -149,11 +152,12 @@ absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
  command_buffer.label = @"InferenceCalculator";
  // Explicit copy input with conversion float 32 bits to 16 bits.
  for (int i = 0; i < input_tensors.size(); ++i) {
-    auto input_view = input_tensors[i].GetMtlBufferReadView(command_buffer);
+    auto input_view =
        MtlBufferView::GetReadView(input_tensors[i], command_buffer);
    // Reshape tensor.
    tflite::gpu::BHWC shape = BhwcFromTensorShape(input_tensors[i].shape());
    auto gpu_buffer_view =
-        gpu_buffers_in_[i]->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*gpu_buffers_in_[i], command_buffer);
    id<MTLComputeCommandEncoder> input_encoder =
        [command_buffer computeCommandEncoder];
    [converter_to_BPHWC4_ convertWithEncoder:input_encoder
@@ -173,9 +177,10 @@ absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
                                 output_shapes_[i]);
    // Reshape tensor.
    tflite::gpu::BHWC shape = BhwcFromTensorShape(output_shapes_[i]);
-    auto read_view = gpu_buffers_out_[i]->GetMtlBufferReadView(command_buffer);
+    auto read_view =
        MtlBufferView::GetReadView(*gpu_buffers_out_[i], command_buffer);
    auto write_view =
-        output_tensors->at(i).GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(output_tensors->at(i), command_buffer);
    id<MTLComputeCommandEncoder> output_encoder =
        [command_buffer computeCommandEncoder];
    [converter_from_BPHWC4_ convertWithEncoder:output_encoder
@@ -224,9 +229,6 @@ absl::Status InferenceCalculatorMetalImpl::InitInterpreter(
 void InferenceCalculatorMetalImpl::AddDelegate(
    CalculatorContext* cc, tflite::InterpreterBuilder* interpreter_builder) {
  const auto& calculator_opts =
      cc->Options<mediapipe::InferenceCalculatorOptions>();
  // Configure and create the delegate.
  TFLGpuDelegateOptions options;
  // `enable_quantization` enables the run of sparse models i.e. the models with
@@ -248,6 +250,9 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
  const auto& input_indices = interpreter_->inputs();
  for (int i = 0; i < input_indices.size(); ++i) {
    const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
        "Input tensor at index [%d] doesn't specify dimensions.",
        input_indices[i]);
    // Create and bind input buffer.
    std::vector<int> dims{tensor->dims->data,
                          tensor->dims->data + tensor->dims->size};
@@ -257,7 +262,7 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
                              : Tensor::ElementType::kFloat32,
        Tensor::Shape{dims}));
    auto buffer_view =
-        gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice);
+        MtlBufferView::GetWriteView(*gpu_buffers_in_[i], gpu_helper_.mtlDevice);
    RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                     delegate_.get(), input_indices[i], buffer_view.buffer()),
                 true);
@@ -269,6 +274,9 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
  output_shapes_.resize(output_indices.size());
  for (int i = 0; i < output_shapes_.size(); ++i) {
    const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
    RET_CHECK(tensor->dims->size > 0) << absl::StrFormat(
        "Output tensor at index [%d] doesn't specify dimensions.",
        output_indices[i]);
    RET_CHECK(tensor->dims->size <= 4);
    // Create and bind output buffers.
    // Channels are always padded to multiple of 4.
@@ -282,8 +290,8 @@ absl::Status InferenceCalculatorMetalImpl::CreateConverters(
        Tensor::Shape{dims}));
    RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                     delegate_.get(), output_indices[i],
-                     gpu_buffers_out_[i]
+                     MtlBufferView::GetWriteView(*gpu_buffers_out_[i],
-                         ->GetMtlBufferWriteView(gpu_helper_.mtlDevice)
+                                                 gpu_helper_.mtlDevice)
                         .buffer()),
                 true);
  }
@@ -70,7 +70,7 @@ absl::Status InferenceCalculatorXnnpackImpl::Process(CalculatorContext* cc) {
  RET_CHECK(!input_tensors.empty());
  ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
-                   inference_runner_->Run(input_tensors));
+                   inference_runner_->Run(cc, input_tensors));
  kOutTensors(cc).Send(std::move(output_tensors));
  return absl::OkStatus();
 }
@@ -114,6 +114,8 @@ InferenceCalculatorXnnpackImpl::CreateDelegate(CalculatorContext* cc) {
  auto xnnpack_opts = TfLiteXNNPackDelegateOptionsDefault();
  xnnpack_opts.num_threads =
      GetXnnpackNumThreads(opts_has_delegate, opts_delegate);
  // TODO Remove once XNNPACK is enabled by default.
  xnnpack_opts.flags |= TFLITE_XNNPACK_DELEGATE_FLAG_QU8;
  return TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts),
                           &TfLiteXNNPackDelegateDelete);
 }
@@ -20,9 +20,14 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/mediapipe_profiling.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "tensorflow/lite/c/c_api_types.h"
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/interpreter_builder.h"
 #include "tensorflow/lite/string_util.h"
 #define PERFETTO_TRACK_EVENT_NAMESPACE mediapipe
 namespace mediapipe {
@@ -39,6 +44,19 @@ void CopyTensorBufferToInterpreter(const Tensor& input_tensor,
  std::memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor.bytes());
 }
 template <>
 void CopyTensorBufferToInterpreter<char>(const Tensor& input_tensor,
                                         tflite::Interpreter* interpreter,
                                         int input_tensor_index) {
  const char* input_tensor_buffer =
      input_tensor.GetCpuReadView().buffer<char>();
  tflite::DynamicBuffer dynamic_buffer;
  dynamic_buffer.AddString(input_tensor_buffer,
                           input_tensor.shape().num_elements());
  dynamic_buffer.WriteToTensorAsVector(
      interpreter->tensor(interpreter->inputs()[input_tensor_index]));
 }
 template <typename T>
 void CopyTensorBufferFromInterpreter(tflite::Interpreter* interpreter,
                                     int output_tensor_index,
@@ -64,7 +82,7 @@ class InferenceInterpreterDelegateRunner : public InferenceRunner {
        delegate_(std::move(delegate)) {}
  absl::StatusOr<std::vector<Tensor>> Run(
-      const std::vector<Tensor>& input_tensors) override;
+      CalculatorContext* cc, const std::vector<Tensor>& input_tensors) override;
 private:
  api2::Packet<TfLiteModelPtr> model_;
@@ -73,7 +91,7 @@ class InferenceInterpreterDelegateRunner : public InferenceRunner {
 };
 absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
-    const std::vector<Tensor>& input_tensors) {
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
  // Read CPU input into tensors.
  RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
  for (int i = 0; i < input_tensors.size(); ++i) {
@@ -87,13 +105,13 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
        break;
      }
      case TfLiteType::kTfLiteUInt8: {
-        CopyTensorBufferToInterpreter<uint8>(input_tensors[i],
+        CopyTensorBufferToInterpreter<uint8_t>(input_tensors[i],
-                                             interpreter_.get(), i);
+                                               interpreter_.get(), i);
        break;
      }
      case TfLiteType::kTfLiteInt8: {
-        CopyTensorBufferToInterpreter<int8>(input_tensors[i],
+        CopyTensorBufferToInterpreter<int8_t>(input_tensors[i],
-                                            interpreter_.get(), i);
+                                              interpreter_.get(), i);
        break;
      }
      case TfLiteType::kTfLiteInt32: {
@@ -101,6 +119,14 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
                                               interpreter_.get(), i);
        break;
      }
      case TfLiteType::kTfLiteString: {
        CopyTensorBufferToInterpreter<char>(input_tensors[i],
                                            interpreter_.get(), i);
        break;
      }
      case TfLiteType::kTfLiteBool:
        // No current use-case for copying MediaPipe Tensors with bool type to
        // TfLiteTensors.
      default:
        return absl::InvalidArgumentError(
            absl::StrCat("Unsupported input tensor type:", input_tensor_type));
@@ -108,8 +134,10 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
  }
  // Run inference.
-  RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+  {
-
+    MEDIAPIPE_PROFILING(CPU_TASK_INVOKE, cc);
    RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
  }
  // Output result tensors (CPU).
  const auto& tensor_indexes = interpreter_->outputs();
  std::vector<Tensor> output_tensors;
@@ -146,6 +174,15 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
        CopyTensorBufferFromInterpreter<int32_t>(interpreter_.get(), i,
                                                 &output_tensors.back());
        break;
      case TfLiteType::kTfLiteBool:
        output_tensors.emplace_back(Tensor::ElementType::kBool, shape,
                                    Tensor::QuantizationParameters{1.0f, 0});
        CopyTensorBufferFromInterpreter<bool>(interpreter_.get(), i,
                                              &output_tensors.back());
        break;
      case TfLiteType::kTfLiteString:
        // No current use-case for copying TfLiteTensors with string type to
        // MediaPipe Tensors.
      default:
        return absl::InvalidArgumentError(
            absl::StrCat("Unsupported output tensor type:",
@@ -2,6 +2,7 @@
 #define MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_RUNNER_H_
 #include "absl/status/statusor.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/formats/tensor.h"
 namespace mediapipe {
@@ -11,7 +12,7 @@ class InferenceRunner {
 public:
  virtual ~InferenceRunner() = default;
  virtual absl::StatusOr<std::vector<Tensor>> Run(
-      const std::vector<Tensor>& inputs) = 0;
+      CalculatorContext* cc, const std::vector<Tensor>& inputs) = 0;
 };
 }  // namespace mediapipe
@@ -0,0 +1,174 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <cstdint>
 #include <cstring>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
 #include "mediapipe/tasks/cc/text/tokenizers/regex_tokenizer.h"
 #include "mediapipe/tasks/cc/text/tokenizers/tokenizer_utils.h"
 #include "mediapipe/tasks/metadata/metadata_schema_generated.h"
 namespace mediapipe {
 namespace api2 {
 using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
 // Preprocesses input text into one int32 input tensor for a text model using
 // a RegexTokenizer.
 //
 // Inputs:
 //   TEXT - std::string
 //     The input text.
 // Side Inputs:
 //   METADATA_EXTRACTOR - ModelMetadataExtractor
 //     The metadata extractor for the text model. Used to extract the metadata
 //     to construct the RegexTokenizer.
 //
 // Outputs:
 //   TENSORS - std::vector<Tensor>
 //     Vector containing a single Tensor which is the text model's input tensor.
 //     Depending on the tokenizer metadata, the tensor may start with
 //     the id of the tokenizer's <START> token. The following tensor values will
 //     be the ids of the tokens of the input text. Any out-of-vocab tokens will
 //     have the id of the <UNKNOWN> token. The tensor will be padded with the
 //     <PAD> token id to have size equal to the max sequence length for the text
 //     model.
 //
 // Example:
 // node {
 //   calculator: "RegexPreprocessorCalculator"
 //   input_stream: "TEXT:text"
 //   input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
 //   output_stream: "TENSORS:tensors"
 //   options {
 //     [mediapipe.RegexPreprocessorCalculatorOptions.ext] {
 //       max_seq_len: 256
 //     }
 //   }
 // }
 class RegexPreprocessorCalculator : public Node {
 public:
  static constexpr Input<std::string> kTextIn{"TEXT"};
  static constexpr SideInput<ModelMetadataExtractor> kMetadataExtractorSideIn{
      "METADATA_EXTRACTOR"};
  static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
  MEDIAPIPE_NODE_CONTRACT(kTextIn, kMetadataExtractorSideIn, kTensorsOut);
  static absl::Status UpdateContract(CalculatorContract* cc);
  absl::Status Open(CalculatorContext* cc) override;
  absl::Status Process(CalculatorContext* cc) override;
 private:
  std::unique_ptr<tasks::text::tokenizers::RegexTokenizer> tokenizer_;
  // The max sequence length accepted by the text model.
  int max_seq_len_ = 0;
 };
 absl::Status RegexPreprocessorCalculator::UpdateContract(
    CalculatorContract* cc) {
  const auto& options =
      cc->Options<mediapipe::RegexPreprocessorCalculatorOptions>();
  RET_CHECK(options.has_max_seq_len()) << "max_seq_len is required";
  RET_CHECK_GT(options.max_seq_len(), 0) << "max_seq_len must be positive";
  return absl::OkStatus();
 }
 absl::Status RegexPreprocessorCalculator::Open(CalculatorContext* cc) {
  const ModelMetadataExtractor* metadata_extractor =
      &kMetadataExtractorSideIn(cc).Get();
  const tflite::TensorMetadata* tensor_metadata =
      metadata_extractor->GetInputTensorMetadata(0);
  if (tensor_metadata == nullptr) {
    return absl::InvalidArgumentError("No tensor metadata found");
  }
  ASSIGN_OR_RETURN(
      const auto* tokenizer_metadata,
      metadata_extractor->FindFirstProcessUnit(
          *tensor_metadata, tflite::ProcessUnitOptions_RegexTokenizerOptions));
  if (tokenizer_metadata == nullptr) {
    return absl::InvalidArgumentError("No tokenizer metadata found");
  }
  const tflite::RegexTokenizerOptions* regex_tokenizer_options =
      tokenizer_metadata->options_as<tflite::RegexTokenizerOptions>();
  ASSIGN_OR_RETURN(tokenizer_,
                   tasks::text::tokenizers::CreateRegexTokenizerFromOptions(
                       regex_tokenizer_options, metadata_extractor));
  const auto& options =
      cc->Options<mediapipe::RegexPreprocessorCalculatorOptions>();
  max_seq_len_ = options.max_seq_len();
  return absl::OkStatus();
 }
 absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
  tasks::text::tokenizers::TokenizerResult tokenizer_result =
      tokenizer_->Tokenize(kTextIn(cc).Get());
  int unknown_token_id = 0;
  tokenizer_->GetUnknownToken(&unknown_token_id);
  int pad_token_id = 0;
  tokenizer_->GetPadToken(&pad_token_id);
  std::vector<int> input_tokens(max_seq_len_, pad_token_id);
  int start_token_id = 0;
  int input_token_index = 0;
  if (tokenizer_->GetStartToken(&start_token_id)) {
    input_tokens[0] = start_token_id;
    input_token_index = 1;
  }
  for (int i = 0; (i < tokenizer_result.subwords.size()) &&
                  (input_token_index < max_seq_len_);
       ++i, ++input_token_index) {
    const std::string& token = tokenizer_result.subwords[i];
    int token_id = 0;
    if (tokenizer_->LookupId(token, &token_id)) {
      input_tokens[input_token_index] = token_id;
    } else {
      input_tokens[input_token_index] = unknown_token_id;
    }
  }
  //                              |<-------sentence_length-------->|
  // input_tensor                 <START>, t1, t2... <PAD>, <PAD>...
  // <START> is optional, t1, t2... will be replaced by <UNKNOWN> if it's
  // not found in the tokenizer vocab.
  std::vector<Tensor> result;
  result.push_back(
      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
  std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
              input_tokens.data(), input_tokens.size() * sizeof(int32_t));
  kTensorsOut(cc).Send(std::move(result));
  return absl::OkStatus();
 }
 MEDIAPIPE_REGISTER_NODE(RegexPreprocessorCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,29 @@
 /* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 message RegexPreprocessorCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional RegexPreprocessorCalculatorOptions ext = 463716697;
  }
  // The maximum input sequence length for the calculator's text model.
  optional int32 max_seq_len = 1;
 }
@@ -0,0 +1,130 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <memory>
 #include <sstream>
 #include <string>
 #include <utility>
 #include <vector>
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/tool/sink.h"
 #include "mediapipe/tasks/cc/core/utils.h"
 #include "mediapipe/tasks/cc/metadata/metadata_extractor.h"
 namespace mediapipe {
 namespace {
 using ::mediapipe::tasks::metadata::ModelMetadataExtractor;
 using ::testing::ElementsAreArray;
 constexpr int kMaxSeqLen = 256;
 constexpr char kTestModelPath[] =
    "mediapipe/tasks/testdata/text/"
    "test_model_text_classifier_with_regex_tokenizer.tflite";
 absl::StatusOr<std::vector<int>> RunRegexPreprocessorCalculator(
    absl::string_view text) {
  auto graph_config =
      ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
          R"pb(
            input_stream: "text"
            output_stream: "tensors"
            node {
              calculator: "RegexPreprocessorCalculator"
              input_stream: "TEXT:text"
              input_side_packet: "METADATA_EXTRACTOR:metadata_extractor"
              output_stream: "TENSORS:tensors"
              options {
                [mediapipe.RegexPreprocessorCalculatorOptions.ext] {
                  max_seq_len: $0
                }
              }
            }
          )pb",
          kMaxSeqLen));
  std::vector<Packet> output_packets;
  tool::AddVectorSink("tensors", &graph_config, &output_packets);
  std::string model_buffer = tasks::core::LoadBinaryContent(kTestModelPath);
  ASSIGN_OR_RETURN(std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
                   ModelMetadataExtractor::CreateFromModelBuffer(
                       model_buffer.data(), model_buffer.size()));
  // Run the graph.
  CalculatorGraph graph;
  MP_RETURN_IF_ERROR(graph.Initialize(
      graph_config,
      {{"metadata_extractor",
        MakePacket<ModelMetadataExtractor>(std::move(*metadata_extractor))}}));
  MP_RETURN_IF_ERROR(graph.StartRun({}));
  MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
      "text", MakePacket<std::string>(text).At(Timestamp(0))));
  MP_RETURN_IF_ERROR(graph.WaitUntilIdle());
  if (output_packets.size() != 1) {
    return absl::InvalidArgumentError(absl::Substitute(
        "output_packets has size $0, expected 1", output_packets.size()));
  }
  const std::vector<Tensor>& tensor_vec =
      output_packets[0].Get<std::vector<Tensor>>();
  if (tensor_vec.size() != 1) {
    return absl::InvalidArgumentError(absl::Substitute(
        "tensor_vec has size $0, expected $1", tensor_vec.size(), 1));
  }
  if (tensor_vec[0].element_type() != Tensor::ElementType::kInt32) {
    return absl::InvalidArgumentError("Expected tensor element type kInt32");
  }
  auto* buffer = tensor_vec[0].GetCpuReadView().buffer<int>();
  std::vector<int> result(buffer, buffer + kMaxSeqLen);
  MP_RETURN_IF_ERROR(graph.CloseAllPacketSources());
  MP_RETURN_IF_ERROR(graph.WaitUntilDone());
  return result;
 }
 TEST(RegexPreprocessorCalculatorTest, TextClassifierModel) {
  MP_ASSERT_OK_AND_ASSIGN(
      std::vector<int> processed_tensor_values,
      RunRegexPreprocessorCalculator("This is the best movie I’ve seen in "
                                     "recent years. Strongly recommend it!"));
  static const int expected_result[kMaxSeqLen] = {
      1, 2, 9, 4, 118, 20, 2, 2, 110, 11, 1136, 153, 2, 386, 12};
  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
 }
 TEST(RegexPreprocessorCalculatorTest, LongInput) {
  std::stringstream long_input;
  long_input << "This is the best";
  for (int i = 0; i < kMaxSeqLen; ++i) {
    long_input << " best";
  }
  long_input << "movie I’ve seen in recent years. Strongly recommend it!";
  MP_ASSERT_OK_AND_ASSIGN(std::vector<int> processed_tensor_values,
                          RunRegexPreprocessorCalculator(long_input.str()));
  std::vector<int> expected_result = {1, 2, 9, 4, 118};
  // "best" id
  expected_result.resize(kMaxSeqLen, 118);
  EXPECT_THAT(processed_tensor_values, ElementsAreArray(expected_result));
 }
 }  // namespace
 }  // namespace mediapipe
@@ -31,6 +31,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 #include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
 #include "mediapipe/gpu/gl_calculator_helper.h"
@@ -296,7 +297,6 @@ absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) {
  output_tensors->emplace_back(Tensor::ElementType::kFloat32,
                               Tensor::Shape{1, height, width, channels});
 #if MEDIAPIPE_METAL_ENABLED
  id<MTLDevice> device = gpu_helper_.mtlDevice;
  id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
  command_buffer.label = @"TensorConverterCalculatorConvert";
  id<MTLComputeCommandEncoder> compute_encoder =
@@ -305,7 +305,7 @@ absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) {
  id<MTLTexture> src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input];
  [compute_encoder setTexture:src_texture atIndex:0];
  auto output_view =
-      output_tensors->at(0).GetMtlBufferWriteView(command_buffer);
+      MtlBufferView::GetWriteView(output_tensors->at(0), command_buffer);
  [compute_encoder setBuffer:output_view.buffer() offset:0 atIndex:1];
  MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1);
  MTLSize threadgroups =
@@ -87,6 +87,9 @@ absl::Status TensorsDequantizationCalculator::Process(CalculatorContext* cc) {
      case Tensor::ElementType::kInt8:
        Dequantize<int8>(input_tensor, &output_tensors->back());
        break;
      case Tensor::ElementType::kBool:
        Dequantize<bool>(input_tensor, &output_tensors->back());
        break;
      default:
        return absl::InvalidArgumentError(absl::StrCat(
            "Unsupported input tensor type: ", input_tensor.element_type()));
@@ -124,5 +124,15 @@ TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithInt8Tensors) {
  ValidateResult(GetOutput(), {-1.007874, 0, 1});
 }
 TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithBoolTensors) {
  std::vector<bool> tensor = {true, false, true};
  PushTensor(Tensor::ElementType::kBool, tensor,
             Tensor::QuantizationParameters{1.0f, 0});
  MP_ASSERT_OK(runner_.Run());
  ValidateResult(GetOutput(), {1, 0, 1});
 }
 }  // namespace
 }  // namespace mediapipe
@@ -0,0 +1,197 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <algorithm>
 #include <cmath>
 #include <cstring>
 #include <new>
 #include <utility>
 #include <vector>
 #include "absl/algorithm/container.h"
 #include "absl/status/status.h"
 #include "audio/dsp/window_functions.h"
 #include "mediapipe/calculators/tensor/tensors_to_audio_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "pffft.h"
 namespace mediapipe {
 namespace api2 {
 namespace {
 std::vector<float> HannWindow(int window_size, bool sqrt_hann) {
  std::vector<float> hann_window(window_size);
  audio_dsp::HannWindow().GetPeriodicSamples(window_size, &hann_window);
  if (sqrt_hann) {
    absl::c_transform(hann_window, hann_window.begin(),
                      [](double x) { return std::sqrt(x); });
  }
  return hann_window;
 }
 // Note that the InvHannWindow function may only work for 50% overlapping case.
 std::vector<float> InvHannWindow(int window_size, bool sqrt_hann) {
  std::vector<float> window = HannWindow(window_size, sqrt_hann);
  std::vector<float> inv_window(window.size());
  if (sqrt_hann) {
    absl::c_copy(window, inv_window.begin());
  } else {
    const int kHalfWindowSize = window.size() / 2;
    absl::c_transform(window, inv_window.begin(),
                      [](double x) { return x * x; });
    for (int i = 0; i < kHalfWindowSize; ++i) {
      double sum = inv_window[i] + inv_window[kHalfWindowSize + i];
      inv_window[i] = window[i] / sum;
      inv_window[kHalfWindowSize + i] = window[kHalfWindowSize + i] / sum;
    }
  }
  return inv_window;
 }
 // PFFFT only supports transforms for inputs of length N of the form
 // N = (2^a)*(3^b)*(5^c) where b >=0 and c >= 0 and a >= 5 for the real FFT.
 bool IsValidFftSize(int size) {
  if (size <= 0) {
    return false;
  }
  constexpr int kFactors[] = {2, 3, 5};
  int factorization[] = {0, 0, 0};
  int n = static_cast<int>(size);
  for (int i = 0; i < 3; ++i) {
    while (n % kFactors[i] == 0) {
      n = n / kFactors[i];
      ++factorization[i];
    }
  }
  return factorization[0] >= 5 && n == 1;
 }
 }  // namespace
 // Converts 2D MediaPipe float Tensors to audio buffers.
 // The calculator will perform ifft on the complex DFT and apply the window
 // function (Inverse Hann) afterwards. The input 2D MediaPipe Tensor must
 // have the DFT real parts in its first row and the DFT imagery parts in its
 // second row. A valid "fft_size" must be set in the CalculatorOptions.
 //
 // Inputs:
 //   TENSORS - std::vector<Tensor>
 //     Vector containing a single Tensor that represents the audio's complex DFT
 //     results.
 //   DC_AND_NYQUIST - std::pair<float, float>
 //     A pair of dc component and nyquist component.
 //
 // Outputs:
 //   AUDIO - mediapipe::Matrix
 //     The audio data represented as mediapipe::Matrix.
 //
 // Example:
 // node {
 //   calculator: "TensorsToAudioCalculator"
 //   input_stream: "TENSORS:tensors"
 //   input_stream: "DC_AND_NYQUIST:dc_and_nyquist"
 //   output_stream: "AUDIO:audio"
 //   options {
 //     [mediapipe.AudioToTensorCalculatorOptions.ext] {
 //       fft_size: 256
 //     }
 //   }
 // }
 class TensorsToAudioCalculator : public Node {
 public:
  static constexpr Input<std::vector<Tensor>> kTensorsIn{"TENSORS"};
  static constexpr Input<std::pair<float, float>> kDcAndNyquistIn{
      "DC_AND_NYQUIST"};
  static constexpr Output<Matrix> kAudioOut{"AUDIO"};
  MEDIAPIPE_NODE_CONTRACT(kTensorsIn, kDcAndNyquistIn, kAudioOut);
  absl::Status Open(CalculatorContext* cc) override;
  absl::Status Process(CalculatorContext* cc) override;
  absl::Status Close(CalculatorContext* cc) override;
 private:
  // The internal state of the FFT library.
  PFFFT_Setup* fft_state_ = nullptr;
  int fft_size_ = 0;
  float inverse_fft_size_ = 0;
  std::vector<float, Eigen::aligned_allocator<float>> input_dft_;
  std::vector<float> inv_fft_window_;
  std::vector<float, Eigen::aligned_allocator<float>> fft_input_buffer_;
  // pffft requires memory to work with to avoid using the stack.
  std::vector<float, Eigen::aligned_allocator<float>> fft_workplace_;
  std::vector<float, Eigen::aligned_allocator<float>> fft_output_;
 };
 absl::Status TensorsToAudioCalculator::Open(CalculatorContext* cc) {
  const auto& options =
      cc->Options<mediapipe::TensorsToAudioCalculatorOptions>();
  RET_CHECK(options.has_fft_size()) << "FFT size must be specified.";
  RET_CHECK(IsValidFftSize(options.fft_size()))
      << "FFT size must be of the form fft_size = (2^a)*(3^b)*(5^c) where b "
         ">=0 and c >= 0 and a >= 5, the requested fft size is "
      << options.fft_size();
  fft_size_ = options.fft_size();
  inverse_fft_size_ = 1.0f / fft_size_;
  fft_state_ = pffft_new_setup(fft_size_, PFFFT_REAL);
  input_dft_.resize(fft_size_);
  inv_fft_window_ = InvHannWindow(fft_size_, /* sqrt_hann = */ false);
  fft_input_buffer_.resize(fft_size_);
  fft_workplace_.resize(fft_size_);
  fft_output_.resize(fft_size_);
  return absl::OkStatus();
 }
 absl::Status TensorsToAudioCalculator::Process(CalculatorContext* cc) {
  if (kTensorsIn(cc).IsEmpty() || kDcAndNyquistIn(cc).IsEmpty()) {
    return absl::OkStatus();
  }
  const auto& input_tensors = *kTensorsIn(cc);
  RET_CHECK_EQ(input_tensors.size(), 1);
  RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
  auto view = input_tensors[0].GetCpuReadView();
  // DC's real part.
  input_dft_[0] = kDcAndNyquistIn(cc)->first;
  // Nyquist's real part is the penultimate element of the tensor buffer.
  // pffft ignores the Nyquist's imagery part. No need to fetch the last value
  // from the tensor buffer.
  input_dft_[1] = *(view.buffer<float>() + (fft_size_ - 2));
  std::memcpy(input_dft_.data() + 2, view.buffer<float>(),
              (fft_size_ - 2) * sizeof(float));
  pffft_transform_ordered(fft_state_, input_dft_.data(), fft_output_.data(),
                          fft_workplace_.data(), PFFFT_BACKWARD);
  // Applies the inverse window function.
  std::transform(
      fft_output_.begin(), fft_output_.end(), inv_fft_window_.begin(),
      fft_output_.begin(),
      [this](float a, float b) { return a * b * inverse_fft_size_; });
  Matrix matrix = Eigen::Map<Matrix>(fft_output_.data(), 1, fft_output_.size());
  kAudioOut(cc).Send(std::move(matrix));
  return absl::OkStatus();
 }
 absl::Status TensorsToAudioCalculator::Close(CalculatorContext* cc) {
  if (fft_state_) {
    pffft_destroy_setup(fft_state_);
  }
  return absl::OkStatus();
 }
 MEDIAPIPE_REGISTER_NODE(TensorsToAudioCalculator);
 }  // namespace api2
 }  // namespace mediapipe
@@ -0,0 +1,29 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 syntax = "proto2";
 package mediapipe;
 import "mediapipe/framework/calculator.proto";
 message TensorsToAudioCalculatorOptions {
  extend mediapipe.CalculatorOptions {
    optional TensorsToAudioCalculatorOptions ext = 484297136;
  }
  // Size of the fft in number of bins. If set, the calculator will do ifft
  // on the input tensor.
  optional int64 fft_size = 1;
 }
@@ -0,0 +1,149 @@
 // Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include <algorithm>
 #include <new>
 #include <string>
 #include <vector>
 #include "absl/status/status.h"
 #include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/audio_to_tensor_calculator.pb.h"
 #include "mediapipe/calculators/tensor/tensors_to_audio_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 namespace mediapipe {
 namespace {
 class TensorsToAudioCalculatorFftTest : public ::testing::Test {
 protected:
  // Creates an audio matrix containing a single sample of 1.0 at a specified
  // offset.
  Matrix CreateImpulseSignalData(int64 num_samples, int impulse_offset_idx) {
    Matrix impulse = Matrix::Zero(1, num_samples);
    impulse(0, impulse_offset_idx) = 1.0;
    return impulse;
  }
  void ConfigGraph(int num_samples, double sample_rate, int fft_size) {
    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
        absl::Substitute(R"(
        input_stream: "audio_in"
        input_stream: "sample_rate"
        output_stream: "audio_out"
        node {
          calculator: "AudioToTensorCalculator"
          input_stream: "AUDIO:audio_in"
          input_stream: "SAMPLE_RATE:sample_rate"
          output_stream: "TENSORS:tensors"
          output_stream: "DC_AND_NYQUIST:dc_and_nyquist"
          options {
            [mediapipe.AudioToTensorCalculatorOptions.ext] {
              num_channels: 1
              num_samples: $0
              num_overlapping_samples: 0
              target_sample_rate: $1
              fft_size: $2
            }
          }
        }
        node {
          calculator: "TensorsToAudioCalculator"
          input_stream: "TENSORS:tensors"
          input_stream: "DC_AND_NYQUIST:dc_and_nyquist"
          output_stream: "AUDIO:audio_out"
          options {
            [mediapipe.TensorsToAudioCalculatorOptions.ext] {
              fft_size: $2
            }
          }
        }
        )",
                         /*$0=*/num_samples,
                         /*$1=*/sample_rate,
                         /*$2=*/fft_size));
    tool::AddVectorSink("audio_out", &graph_config_, &audio_out_packets_);
  }
  void RunGraph(const Matrix& input_data, double sample_rate) {
    MP_ASSERT_OK(graph_.Initialize(graph_config_));
    MP_ASSERT_OK(graph_.StartRun({}));
    MP_ASSERT_OK(graph_.AddPacketToInputStream(
        "sample_rate", MakePacket<double>(sample_rate).At(Timestamp(0))));
    MP_ASSERT_OK(graph_.AddPacketToInputStream(
        "audio_in", MakePacket<Matrix>(input_data).At(Timestamp(0))));
    MP_ASSERT_OK(graph_.CloseAllInputStreams());
    MP_ASSERT_OK(graph_.WaitUntilDone());
  }
  std::vector<Packet> audio_out_packets_;
  CalculatorGraphConfig graph_config_;
  CalculatorGraph graph_;
 };
 TEST_F(TensorsToAudioCalculatorFftTest, TestInvalidFftSize) {
  ConfigGraph(320, 16000, 103);
  MP_ASSERT_OK(graph_.Initialize(graph_config_));
  MP_ASSERT_OK(graph_.StartRun({}));
  auto status = graph_.WaitUntilIdle();
  EXPECT_EQ(status.code(), absl::StatusCode::kInternal);
  EXPECT_THAT(status.message(),
              ::testing::HasSubstr("FFT size must be of the form"));
 }
 TEST_F(TensorsToAudioCalculatorFftTest, TestImpulseSignalAtTheCenter) {
  constexpr int sample_size = 320;
  constexpr double sample_rate = 16000;
  ConfigGraph(sample_size, sample_rate, 320);
  Matrix impulse_data = CreateImpulseSignalData(sample_size, sample_size / 2);
  RunGraph(impulse_data, sample_rate);
  ASSERT_EQ(1, audio_out_packets_.size());
  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
  // The impulse signal at the center is not affected by the window function.
  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), impulse_data);
 }
 TEST_F(TensorsToAudioCalculatorFftTest, TestWindowedImpulseSignal) {
  constexpr int sample_size = 320;
  constexpr double sample_rate = 16000;
  ConfigGraph(sample_size, sample_rate, 320);
  Matrix impulse_data = CreateImpulseSignalData(sample_size, sample_size / 4);
  RunGraph(impulse_data, sample_rate);
  ASSERT_EQ(1, audio_out_packets_.size());
  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
  // As the impulse signal sits at the 1/4 of the hann window, the inverse
  // window function reduces it by half.
  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), impulse_data / 2);
 }
 TEST_F(TensorsToAudioCalculatorFftTest, TestImpulseSignalAtBeginning) {
  constexpr int sample_size = 320;
  constexpr double sample_rate = 16000;
  ConfigGraph(sample_size, sample_rate, 320);
  Matrix impulse_data = CreateImpulseSignalData(sample_size, 0);
  RunGraph(impulse_data, sample_rate);
  ASSERT_EQ(1, audio_out_packets_.size());
  MP_ASSERT_OK(audio_out_packets_[0].ValidateAsType<Matrix>());
  // As the impulse signal sits at the beginning of the hann window, the inverse
  // window function completely removes it.
  EXPECT_EQ(audio_out_packets_[0].Get<Matrix>(), Matrix::Zero(1, sample_size));
 }
 }  // namespace
 }  // namespace mediapipe
@@ -41,6 +41,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 #include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
@@ -532,15 +533,15 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
                                         detection_classes.data(),
                                         output_detections));
 #elif MEDIAPIPE_METAL_ENABLED
  id<MTLDevice> device = gpu_helper_.mtlDevice;
  if (!anchors_init_) {
    if (input_tensors.size() == kNumInputTensorsWithAnchors) {
      RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors);
      auto command_buffer = [gpu_helper_ commandBuffer];
-      auto src_buffer = input_tensors[tensor_mapping_.anchors_tensor_index()]
+      auto src_buffer = MtlBufferView::GetReadView(
-                            .GetMtlBufferReadView(command_buffer);
+          input_tensors[tensor_mapping_.anchors_tensor_index()],
          command_buffer);
      auto dest_buffer =
-          raw_anchors_buffer_->GetMtlBufferWriteView(command_buffer);
+          MtlBufferView::GetWriteView(*raw_anchors_buffer_, command_buffer);
      id<MTLBlitCommandEncoder> blit_command =
          [command_buffer blitCommandEncoder];
      [blit_command copyFromBuffer:src_buffer.buffer()
@@ -572,15 +573,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
  [command_encoder setComputePipelineState:decode_program_];
  {
    auto scored_boxes_view =
-        scored_boxes_buffer_->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*scored_boxes_buffer_, command_buffer);
    auto decoded_boxes_view =
-        decoded_boxes_buffer_->GetMtlBufferWriteView(command_buffer);
+        MtlBufferView::GetWriteView(*decoded_boxes_buffer_, command_buffer);
    [command_encoder setBuffer:decoded_boxes_view.buffer() offset:0 atIndex:0];
-    auto input0_view = input_tensors[tensor_mapping_.detections_tensor_index()]
+    auto input0_view = MtlBufferView::GetReadView(
-                           .GetMtlBufferReadView(command_buffer);
+        input_tensors[tensor_mapping_.detections_tensor_index()],
        command_buffer);
    [command_encoder setBuffer:input0_view.buffer() offset:0 atIndex:1];
    auto raw_anchors_view =
-        raw_anchors_buffer_->GetMtlBufferReadView(command_buffer);
+        MtlBufferView::GetReadView(*raw_anchors_buffer_, command_buffer);
    [command_encoder setBuffer:raw_anchors_view.buffer() offset:0 atIndex:2];
    MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1);
    MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
@@ -589,8 +591,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
    [command_encoder setComputePipelineState:score_program_];
    [command_encoder setBuffer:scored_boxes_view.buffer() offset:0 atIndex:0];
-    auto input1_view = input_tensors[tensor_mapping_.scores_tensor_index()]
+    auto input1_view = MtlBufferView::GetReadView(
-                           .GetMtlBufferReadView(command_buffer);
+        input_tensors[tensor_mapping_.scores_tensor_index()], command_buffer);
    [command_encoder setBuffer:input1_view.buffer() offset:0 atIndex:1];
    MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1);
    MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1);
@@ -53,6 +53,7 @@
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
 #include "mediapipe/framework/formats/tensor_mtl_buffer_view.h"
 #import "mediapipe/gpu/MPPMetalHelper.h"
 #include "mediapipe/gpu/MPPMetalUtil.h"
 #endif  // MEDIAPIPE_METAL_ENABLED
@@ -485,7 +486,8 @@ absl::Status TensorsToSegmentationCalculator::ProcessGpu(
        [command_buffer computeCommandEncoder];
    [command_encoder setComputePipelineState:mask_program_];
-    auto read_view = input_tensors[0].GetMtlBufferReadView(command_buffer);
+    auto read_view =
        MtlBufferView::GetReadView(input_tensors[0], command_buffer);
    [command_encoder setBuffer:read_view.buffer() offset:0 atIndex:0];
    mediapipe::GpuBuffer small_mask_buffer = [metal_helper_
--- a/Show More
+++ b/Show More