Project import generated by Copybara.

GitOrigin-RevId: 791df031256eb214ce0e33faad039133c0061362

.bazelrc

@@ -3,7 +3,7 @@
 # Basic build settings
 build --jobs 128
 build --define='absl=1'
-build --cxxopt='-std=c++11'
+build --cxxopt='-std=c++14'
 build --copt='-Wno-sign-compare'
 build --copt='-Wno-unused-function'
 build --copt='-Wno-uninitialized'
@@ -14,6 +14,7 @@ build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
 # Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
 build --incompatible_disable_deprecated_attr_params=false
+build --incompatible_depset_is_not_iterable=false
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
@@ -34,3 +35,30 @@ build:android_arm --fat_apk_cpu=armeabi-v7a
 build:android_arm64 --config=android
 build:android_arm64 --cpu=arm64-v8a
 build:android_arm64 --fat_apk_cpu=arm64-v8a
+
+# iOS configs.
+build:ios --apple_platform_type=ios
+
+build:ios_i386 --config=ios
+build:ios_i386 --cpu=ios_i386
+build:ios_i386 --watchos_cpus=i386
+
+build:ios_x86_64 --config=ios
+build:ios_x86_64 --cpu=ios_x86_64
+build:ios_x86_64 --watchos_cpus=i386
+
+build:ios_armv7 --config=ios
+build:ios_armv7 --cpu=ios_armv7
+build:ios_armv7 --watchos_cpus=armv7k
+
+build:ios_arm64 --config=ios
+build:ios_arm64 --cpu=ios_arm64
+build:ios_arm64 --watchos_cpus=armv7k
+
+build:ios_arm64e --config=ios
+build:ios_arm64e --cpu=ios_arm64e
+build:ios_arm64e --watchos_cpus=armv7k
+
+build:ios_fat --config=ios
+build:ios_fat --ios_multi_cpus=armv7,arm64
+build:ios_fat --watchos_cpus=armv7k

.gitignore

@@ -0,0 +1,8 @@
+mediapipe/provisioning_profile.mobileprovision
+bazel-bin
+bazel-genfiles
+bazel-mediapipe-ioss
+bazel-out
+bazel-testlogs
+mediapipe/MediaPipe.xcodeproj
+mediapipe/MediaPipe.tulsiproj/*.tulsiconf-user

BUILD

@@ -1,4 +1,4 @@
-# Copyright 2019 The MediaPipeOSS Authors.
+# Copyright 2019 The MediaPipe Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

Dockerfile

@@ -24,18 +24,25 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         ca-certificates \
+        curl \
         git \
         wget \
         unzip \
         python \
+        python-pip \
         libopencv-core-dev \
         libopencv-highgui-dev \
         libopencv-imgproc-dev \
         libopencv-video-dev \
-        && \
+        software-properties-common && \
+    add-apt-repository -y ppa:openjdk-r/ppa && \
+    apt-get update && apt-get install -y openjdk-8-jdk && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+RUN pip install --upgrade setuptools
+RUN pip install future
+
 # Install bazel
 ARG BAZEL_VERSION=0.26.1
 RUN mkdir /bazel && \
@@ -49,4 +56,4 @@ azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
 COPY . /mediapipe/
 
 # If we want the docker image to contain the pre-built object_detection_offline_demo binary, do the following
-# RUN bazel build -c opt --define 'MEDIAPIPE_DISABLE_GPU=1' mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
+# RUN bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/demo:object_detection_tensorflow_demo

README.md

@@ -5,11 +5,27 @@
 
 ![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)
 
+> "<em>MediaPipe has made it extremely easy to build our 3D person pose reconstruction demo app, facilitating accelerated neural network inference on device and synchronization of our result visualization with the video capture stream. Highly recommended!</em>" - George Papandreou, CTO, [Ariel AI](https://arielai.com)
+
+## ML Solutions in MediaPipe
+
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
+* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
+* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
+
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
+![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
+![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
+![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
+
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).
 
 ## Getting started
-See mobile and desktop [examples](mediapipe/docs/examples.md).
+See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
 
 ## Documentation
 [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
@@ -20,16 +36,25 @@ Check out the [Examples page](https://mediapipe.readthedocs.io/en/latest/example
 A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.dev/). Please also see instructions [here](mediapipe/docs/visualizer.md).
 
 ## Community forum
-*  [discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
+*  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
 
 ## Publications
+* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
-[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
+* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
+* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
+* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
+* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
+* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
+* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
+* [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
+* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
+* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
 
 ## Alpha Disclaimer
-MediaPipe is currently in alpha for v0.5. We are still making breaking API changes and expect to get to stable API by v1.0.
+MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.
 
 ## Contributing
 We welcome contributions. Please follow these [guidelines](./CONTRIBUTING.md).

WORKSPACE

@@ -2,14 +2,15 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+skylib_version = "0.8.0"
 http_archive(
     name = "bazel_skylib",
-    sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
+    type = "tar.gz",
-    strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
+    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
-    urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
+    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.23.0")
+versions.check(minimum_bazel_version = "0.24.1")
 
 # ABSL cpp library.
 http_archive(
@@ -24,6 +25,12 @@ http_archive(
     strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
 )
 
+http_archive(
+    name = "rules_cc",
+    strip_prefix = "rules_cc-master",
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
+)
+
 # GoogleTest/GoogleMock framework. Used by most unit-tests.
 http_archive(
      name = "com_google_googletest",
@@ -52,11 +59,17 @@ http_archive(
 
 # glog
 http_archive(
-    name = "com_google_glog",
+    name = "com_github_glog_glog",
     url = "https://github.com/google/glog/archive/v0.3.5.zip",
     sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
     strip_prefix = "glog-0.3.5",
     build_file = "@//third_party:glog.BUILD",
+    patches = [
+        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
 )
 
 # libyuv
@@ -73,6 +86,12 @@ http_archive(
     urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
 )
 
+http_archive(
+    name = "com_google_audio_tools",
+    strip_prefix = "multichannel-audio-tools-master",
+    urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
+)
+
 # Needed by TensorFlow
 http_archive(
     name = "io_bazel_rules_closure",
@@ -84,38 +103,68 @@ http_archive(
     ],
 )
 
-# TensorFlow r1.14-rc0
+# 2019-11-21
+_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
+_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
 http_archive(
     name = "org_tensorflow",
-    strip_prefix = "tensorflow-1.14.0-rc0",
+    urls = [
-    sha256 = "76404a6157a45e8d7a07e4f5690275256260130145924c2a7c73f6eda2a3de10",
+      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
-    urls = ["https://github.com/tensorflow/tensorflow/archive/v1.14.0-rc0.zip"],
+      "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+    ],
+    strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
+    sha256 = _TENSORFLOW_SHA256,
 )
 
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
 tf_workspace(tf_repo_name = "org_tensorflow")
 
-# Please run $ sudo apt-get install libopencv-dev
+# Please run
+# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+#                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
     build_file = "@//third_party:opencv_linux.BUILD",
     path = "/usr",
 )
 
-# Please run $ brew install opencv
+new_local_repository(
+    name = "linux_ffmpeg",
+    build_file = "@//third_party:ffmpeg_linux.BUILD",
+    path = "/usr"
+)
+
+# Please run $ brew install opencv@3
 new_local_repository(
     name = "macos_opencv",
     build_file = "@//third_party:opencv_macos.BUILD",
     path = "/usr",
 )
 
+new_local_repository(
+    name = "macos_ffmpeg",
+    build_file = "@//third_party:ffmpeg_macos.BUILD",
+    path = "/usr",
+)
+
 http_archive(
     name = "android_opencv",
-    sha256="056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
     build_file = "@//third_party:opencv_android.BUILD",
     strip_prefix = "OpenCV-android-sdk",
     type = "zip",
-    url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
+    url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
+)
+
+# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
+# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
+# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
+# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
+http_archive(
+    name = "ios_opencv",
+    sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
+    build_file = "@//third_party:opencv_ios.BUILD",
+    type = "zip",
+    url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
 RULES_JVM_EXTERNAL_TAG = "2.2"
@@ -132,11 +181,19 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")
 
 maven_install(
     artifacts = [
-        "com.android.support.constraint:constraint-layout:aar:1.0.2",
+        "androidx.annotation:annotation:aar:1.1.0",
-        "androidx.appcompat:appcompat:aar:1.0.2",
+        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
+        "androidx.camera:camera-core:aar:1.0.0-alpha06",
+        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
+        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
+        "androidx.core:core:aar:1.1.0-rc03",
+        "androidx.legacy:legacy-support-v4:aar:1.0.0",
+        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
+        "com.google.android.material:material:aar:1.0.0-rc01",
     ],
     repositories = [
         "https://dl.google.com/dl/android/maven2",
+        "https://repo1.maven.org/maven2",
     ],
 )
 
@@ -153,10 +210,10 @@ maven_jar(
 )
 
 maven_jar(
-     name = "androidx_concurrent_futures",
+    name = "androidx_concurrent_futures",
-     artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
+    artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
-     sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
+    sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
-     server = "google_server",
+    server = "google_server",
 )
 
 maven_jar(
@@ -191,3 +248,43 @@ android_ndk_repository(
 android_sdk_repository(
     name = "androidsdk",
 )
+
+# iOS basic build deps.
+
+http_archive(
+    name = "build_bazel_rules_apple",
+    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
+    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
+    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
+)
+
+load(
+    "@build_bazel_rules_apple//apple:repositories.bzl",
+    "apple_rules_dependencies",
+)
+
+apple_rules_dependencies()
+
+load(
+    "@build_bazel_rules_swift//swift:repositories.bzl",
+    "swift_rules_dependencies",
+)
+
+swift_rules_dependencies()
+
+load(
+    "@build_bazel_apple_support//lib:repositories.bzl",
+    "apple_support_dependencies",
+)
+
+apple_support_dependencies()
+
+# More iOS deps.
+
+http_archive(
+    name = "google_toolbox_for_mac",
+    url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
+    sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
+    strip_prefix = "google-toolbox-for-mac-2.2.1",
+    build_file = "@//third_party:google_toolbox_for_mac.BUILD",
+)

mediapipe/BUILD

@@ -65,11 +65,73 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-config_setting(
+# Note: this cannot just match "apple_platform_type": "macos" because that option
+# defaults to "macos" even when building on Linux!
+alias(
     name = "macos",
+    actual = select({
+        ":macos_i386": ":macos_i386",
+        ":macos_x86_64": ":macos_x86_64",
+        "//conditions:default": ":macos_i386",  # Arbitrarily chosen from above.
+    }),
+    visibility = ["//visibility:public"],
+)
+
+# Note: this also matches on crosstool_top so that it does not produce ambiguous
+# selectors when used together with "android".
+config_setting(
+    name = "ios",
+    values = {
+        "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
+        "apple_platform_type": "ios",
+    },
+    visibility = ["//visibility:public"],
+)
+
+alias(
+    name = "apple",
+    actual = select({
+        ":macos": ":macos",
+        ":ios": ":ios",
+        "//conditions:default": ":ios",  # Arbitrarily chosen from above.
+    }),
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "macos_i386",
     values = {
         "apple_platform_type": "macos",
         "cpu": "darwin",
     },
     visibility = ["//visibility:public"],
 )
+
+config_setting(
+    name = "macos_x86_64",
+    values = {
+        "apple_platform_type": "macos",
+        "cpu": "darwin_x86_64",
+    },
+    visibility = ["//visibility:public"],
+)
+
+[
+    config_setting(
+        name = arch,
+        values = {"cpu": arch},
+        visibility = ["//visibility:public"],
+    )
+    for arch in [
+        "ios_i386",
+        "ios_x86_64",
+        "ios_armv7",
+        "ios_arm64",
+        "ios_arm64e",
+    ]
+]
+
+exports_files(
+    ["provisioning_profile.mobileprovision"],
+    visibility = ["//visibility:public"],
+)

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -0,0 +1,115 @@
+{
+  "additionalFilePaths" : [
+    "/BUILD",
+    "mediapipe/BUILD",
+    "mediapipe/objc/BUILD",
+    "mediapipe/examples/ios/BUILD",
+    "mediapipe/examples/ios/edgedetectiongpu/BUILD",
+    "mediapipe/examples/ios/facedetectioncpu/BUILD",
+    "mediapipe/examples/ios/facedetectiongpu/BUILD",
+    "mediapipe/examples/ios/handdetectiongpu/BUILD",
+    "mediapipe/examples/ios/handtrackinggpu/BUILD",
+    "mediapipe/examples/ios/objectdetectioncpu/BUILD",
+    "mediapipe/examples/ios/objectdetectiongpu/BUILD"
+  ],
+  "buildTargets" : [
+    "//mediapipe/examples/ios/edgedetectiongpu:EdgeDetectionGpuApp",
+    "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
+    "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
+    "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
+    "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
+    "//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp",
+    "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp",
+    "//mediapipe/objc:mediapipe_framework_ios"
+  ],
+  "optionSet" : {
+    "BazelBuildOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsDebug" : {
+      "p" : "$(inherited)"
+    },
+    "BazelBuildStartupOptionsRelease" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "BuildActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "CommandlineArguments" : {
+      "p" : "$(inherited)"
+    },
+    "EnvironmentVariables" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "LaunchActionPreActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "ProjectGenerationBazelStartupOptions" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPostActionScript" : {
+      "p" : "$(inherited)"
+    },
+    "TestActionPreActionScript" : {
+      "p" : "$(inherited)"
+    }
+  },
+  "projectName" : "Mediapipe",
+  "sourceFilters" : [
+    "mediapipe",
+    "mediapipe/calculators",
+    "mediapipe/calculators/core",
+    "mediapipe/calculators/image",
+    "mediapipe/calculators/internal",
+    "mediapipe/calculators/tflite",
+    "mediapipe/calculators/util",
+    "mediapipe/examples",
+    "mediapipe/examples/ios",
+    "mediapipe/examples/ios/edgedetectiongpu",
+    "mediapipe/examples/ios/edgedetectiongpu/Base.lproj",
+    "mediapipe/examples/ios/facedetectioncpu",
+    "mediapipe/examples/ios/facedetectioncpu/Base.lproj",
+    "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/facedetectiongpu/Base.lproj",
+    "mediapipe/examples/ios/handdetectiongpu",
+    "mediapipe/examples/ios/handdetectiongpu/Base.lproj",
+    "mediapipe/examples/ios/handtrackinggpu",
+    "mediapipe/examples/ios/handtrackinggpu/Base.lproj",
+    "mediapipe/examples/ios/objectdetectioncpu",
+    "mediapipe/examples/ios/objectdetectioncpu/Base.lproj",
+    "mediapipe/examples/ios/objectdetectiongpu",
+    "mediapipe/examples/ios/objectdetectiongpu/Base.lproj",
+    "mediapipe/framework",
+    "mediapipe/framework/deps",
+    "mediapipe/framework/formats",
+    "mediapipe/framework/formats/annotation",
+    "mediapipe/framework/formats/object_detection",
+    "mediapipe/framework/port",
+    "mediapipe/framework/profiler",
+    "mediapipe/framework/stream_handler",
+    "mediapipe/framework/tool",
+    "mediapipe/gpu",
+    "mediapipe/graphs",
+    "mediapipe/graphs/edge_detection",
+    "mediapipe/graphs/face_detection",
+    "mediapipe/graphs/hand_tracking",
+    "mediapipe/graphs/object_detection",
+    "mediapipe/models",
+    "mediapipe/objc",
+    "mediapipe/util",
+    "mediapipe/util/android",
+    "mediapipe/util/android/file",
+    "mediapipe/util/android/file/base",
+    "mediapipe/util/tflite",
+    "mediapipe/util/tflite/operations"
+  ]
+}

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -0,0 +1,24 @@
+{
+  "configDefaults" : {
+    "optionSet" : {
+      "CLANG_CXX_LANGUAGE_STANDARD" : {
+        "p" : "c++14"
+      }
+    }
+  },
+  "packages" : [
+    "",
+    "mediapipe",
+    "mediapipe/objc",
+    "mediapipe/examples/ios",
+    "mediapipe/examples/ios/edgedetectiongpu",
+    "mediapipe/examples/ios/facedetectioncpu",
+    "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/handdetectiongpu",
+    "mediapipe/examples/ios/handtrackinggpu",
+    "mediapipe/examples/ios/objectdetectioncpu",
+    "mediapipe/examples/ios/objectdetectiongpu"
+  ],
+  "projectName" : "Mediapipe",
+  "workspaceRoot" : "../.."
+}

mediapipe/calculators/audio/BUILD

@@ -0,0 +1,356 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
+proto_library(
+    name = "mfcc_mel_calculators_proto",
+    srcs = ["mfcc_mel_calculators.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "mfcc_mel_calculators_cc_proto",
+    srcs = ["mfcc_mel_calculators.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":mfcc_mel_calculators_proto"],
+)
+
+proto_library(
+    name = "rational_factor_resample_calculator_proto",
+    srcs = ["rational_factor_resample_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "rational_factor_resample_calculator_cc_proto",
+    srcs = ["rational_factor_resample_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":rational_factor_resample_calculator_proto"],
+)
+
+proto_library(
+    name = "spectrogram_calculator_proto",
+    srcs = ["spectrogram_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "spectrogram_calculator_cc_proto",
+    srcs = ["spectrogram_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":spectrogram_calculator_proto"],
+)
+
+proto_library(
+    name = "stabilized_log_calculator_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "stabilized_log_calculator_cc_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":stabilized_log_calculator_proto"],
+)
+
+proto_library(
+    name = "time_series_framer_calculator_proto",
+    srcs = ["time_series_framer_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "time_series_framer_calculator_cc_proto",
+    srcs = ["time_series_framer_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":time_series_framer_calculator_proto"],
+)
+
+cc_library(
+    name = "audio_decoder_calculator",
+    srcs = ["audio_decoder_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:audio_decoder",
+        "//mediapipe/util:audio_decoder_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "basic_time_series_calculators",
+    srcs = ["basic_time_series_calculators.cc"],
+    hdrs = ["basic_time_series_calculators.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/strings",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "mfcc_mel_calculators",
+    srcs = ["mfcc_mel_calculators.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":mfcc_mel_calculators_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/strings",
+        "@com_google_audio_tools//audio/dsp/mfcc",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "rational_factor_resample_calculator",
+    srcs = ["rational_factor_resample_calculator.cc"],
+    hdrs = ["rational_factor_resample_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":rational_factor_resample_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/strings",
+        "@com_google_audio_tools//audio/dsp:resampler",
+        "@com_google_audio_tools//audio/dsp:resampler_rational_factor",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "stabilized_log_calculator",
+    srcs = ["stabilized_log_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "spectrogram_calculator",
+    srcs = ["spectrogram_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":spectrogram_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:source_location",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/strings",
+        "@com_google_audio_tools//audio/dsp:window_functions",
+        "@com_google_audio_tools//audio/dsp/spectrogram",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "time_series_framer_calculator",
+    srcs = ["time_series_framer_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+        "@com_google_audio_tools//audio/dsp:window_functions",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "audio_decoder_calculator_test",
+    srcs = ["audio_decoder_calculator_test.cc"],
+    data = ["//mediapipe/calculators/audio/testdata:test_audios"],
+    deps = [
+        ":audio_decoder_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+    ],
+)
+
+cc_test(
+    name = "basic_time_series_calculators_test",
+    srcs = ["basic_time_series_calculators_test.cc"],
+    deps = [
+        ":basic_time_series_calculators",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/util:time_series_test_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "mfcc_mel_calculators_test",
+    srcs = ["mfcc_mel_calculators_test.cc"],
+    deps = [
+        ":mfcc_mel_calculators",
+        ":mfcc_mel_calculators_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "spectrogram_calculator_test",
+    srcs = ["spectrogram_calculator_test.cc"],
+    deps = [
+        ":spectrogram_calculator",
+        ":spectrogram_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:benchmark",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@com_google_audio_tools//audio/dsp:number_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "stabilized_log_calculator_test",
+    srcs = ["stabilized_log_calculator_test.cc"],
+    deps = [
+        ":stabilized_log_calculator",
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "time_series_framer_calculator_test",
+    srcs = ["time_series_framer_calculator_test.cc"],
+    deps = [
+        ":time_series_framer_calculator",
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@com_google_audio_tools//audio/dsp:window_functions",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "rational_factor_resample_calculator_test",
+    srcs = ["rational_factor_resample_calculator_test.cc"],
+    deps = [
+        ":rational_factor_resample_calculator",
+        ":rational_factor_resample_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+        "//mediapipe/util:time_series_test_util",
+        "@com_google_audio_tools//audio/dsp:signal_vector_util",
+        "@eigen_archive//:eigen",
+    ],
+)

mediapipe/calculators/audio/audio_decoder_calculator.cc

@@ -0,0 +1,110 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/audio_decoder.h"
+#include "mediapipe/util/audio_decoder.pb.h"
+
+namespace mediapipe {
+
+// The AudioDecoderCalculator decodes an audio stream of the media file. It
+// produces two output streams contain audio packets and the header infomation.
+//
+// Output Streams:
+//   AUDIO: Output audio frames (Matrix).
+//   AUDIO_HEADER:
+//       Optional audio header information output
+// Input Side Packets:
+//   INPUT_FILE_PATH: The input file path.
+//
+// Example config:
+// node {
+//   calculator: "AudioDecoderCalculator"
+//   input_side_packet: "INPUT_FILE_PATH:input_file_path"
+//   output_stream: "AUDIO:audio"
+//   output_stream: "AUDIO_HEADER:audio_header"
+//   node_options {
+//     [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
+//        audio_stream { stream_index: 0 }
+//        start_time: 0
+//        end_time: 1
+//   }
+// }
+//
+// TODO: support decoding multiple streams.
+class AudioDecoderCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  std::unique_ptr<AudioDecoder> decoder_;
+};
+
+::mediapipe::Status AudioDecoderCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
+  if (cc->InputSidePackets().HasTag("OPTIONS")) {
+    cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
+  }
+  cc->Outputs().Tag("AUDIO").Set<Matrix>();
+  if (cc->Outputs().HasTag("AUDIO_HEADER")) {
+    cc->Outputs().Tag("AUDIO_HEADER").SetNone();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
+  const std::string& input_file_path =
+      cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
+  const auto& decoder_options =
+      tool::RetrieveOptions(cc->Options<mediapipe::AudioDecoderOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
+  decoder_ = absl::make_unique<AudioDecoder>();
+  MP_RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options));
+  std::unique_ptr<mediapipe::TimeSeriesHeader> header =
+      absl::make_unique<mediapipe::TimeSeriesHeader>();
+  if (decoder_->FillAudioHeader(decoder_options.audio_stream(0), header.get())
+          .ok()) {
+    // Only pass on a header if the decoder could actually produce one.
+    // otherwise, the header will be empty.
+    cc->Outputs().Tag("AUDIO_HEADER").SetHeader(Adopt(header.release()));
+  }
+  cc->Outputs().Tag("AUDIO_HEADER").Close();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status AudioDecoderCalculator::Process(CalculatorContext* cc) {
+  Packet data;
+  int options_index = -1;
+  auto status = decoder_->GetData(&options_index, &data);
+  if (status.ok()) {
+    cc->Outputs().Tag("AUDIO").AddPacket(data);
+  }
+  return status;
+}
+
+::mediapipe::Status AudioDecoderCalculator::Close(CalculatorContext* cc) {
+  return decoder_->Close();
+}
+
+REGISTER_CALCULATOR(AudioDecoderCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -0,0 +1,149 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+TEST(AudioDecoderCalculatorTest, TestWAV) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "AudioDecoderCalculator"
+        input_side_packet: "INPUT_FILE_PATH:input_file_path"
+        output_stream: "AUDIO:audio"
+        output_stream: "AUDIO_HEADER:audio_header"
+        node_options {
+          [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
+            audio_stream { stream_index: 0 }
+          }
+        })");
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
+      file::JoinPath("./",
+                     "/mediapipe/calculators/audio/"
+                     "testdata/sine_wave_1k_44100_mono_2_sec_wav.audio"));
+  MP_ASSERT_OK(runner.Run());
+  MP_EXPECT_OK(runner.Outputs()
+                   .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
+  const mediapipe::TimeSeriesHeader& header =
+      runner.Outputs()
+          .Tag("AUDIO_HEADER")
+          .header.Get<mediapipe::TimeSeriesHeader>();
+  EXPECT_EQ(44100, header.sample_rate());
+  EXPECT_EQ(1, header.num_channels());
+  EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
+              std::ceil(44100.0 * 2 / 2048));
+}
+
+TEST(AudioDecoderCalculatorTest, Test48KWAV) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "AudioDecoderCalculator"
+        input_side_packet: "INPUT_FILE_PATH:input_file_path"
+        output_stream: "AUDIO:audio"
+        output_stream: "AUDIO_HEADER:audio_header"
+        node_options {
+          [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
+            audio_stream { stream_index: 0 }
+          }
+        })");
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
+      file::JoinPath("./",
+                     "/mediapipe/calculators/audio/"
+                     "testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio"));
+  MP_ASSERT_OK(runner.Run());
+  MP_EXPECT_OK(runner.Outputs()
+                   .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
+  const mediapipe::TimeSeriesHeader& header =
+      runner.Outputs()
+          .Tag("AUDIO_HEADER")
+          .header.Get<mediapipe::TimeSeriesHeader>();
+  EXPECT_EQ(48000, header.sample_rate());
+  EXPECT_EQ(2, header.num_channels());
+  EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
+              std::ceil(48000.0 * 2 / 1024));
+}
+
+TEST(AudioDecoderCalculatorTest, TestMP3) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "AudioDecoderCalculator"
+        input_side_packet: "INPUT_FILE_PATH:input_file_path"
+        output_stream: "AUDIO:audio"
+        output_stream: "AUDIO_HEADER:audio_header"
+        node_options {
+          [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
+            audio_stream { stream_index: 0 }
+          }
+        })");
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
+      file::JoinPath("./",
+                     "/mediapipe/calculators/audio/"
+                     "testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio"));
+  MP_ASSERT_OK(runner.Run());
+  MP_EXPECT_OK(runner.Outputs()
+                   .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
+  const mediapipe::TimeSeriesHeader& header =
+      runner.Outputs()
+          .Tag("AUDIO_HEADER")
+          .header.Get<mediapipe::TimeSeriesHeader>();
+  EXPECT_EQ(44100, header.sample_rate());
+  EXPECT_EQ(2, header.num_channels());
+  EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
+              std::ceil(44100.0 * 2 / 1152));
+}
+
+TEST(AudioDecoderCalculatorTest, TestAAC) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "AudioDecoderCalculator"
+        input_side_packet: "INPUT_FILE_PATH:input_file_path"
+        output_stream: "AUDIO:audio"
+        output_stream: "AUDIO_HEADER:audio_header"
+        node_options {
+          [type.googleapis.com/mediapipe.AudioDecoderOptions]: {
+            audio_stream { stream_index: 0 }
+          }
+        })");
+  CalculatorRunner runner(node_config);
+  runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket<std::string>(
+      file::JoinPath("./",
+                     "/mediapipe/calculators/audio/"
+                     "testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio"));
+  MP_ASSERT_OK(runner.Run());
+  MP_EXPECT_OK(runner.Outputs()
+                   .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
+  const mediapipe::TimeSeriesHeader& header =
+      runner.Outputs()
+          .Tag("AUDIO_HEADER")
+          .header.Get<mediapipe::TimeSeriesHeader>();
+  EXPECT_EQ(44100, header.sample_rate());
+  EXPECT_EQ(2, header.num_channels());
+  EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >=
+              std::ceil(44100.0 * 2 / 1024));
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/basic_time_series_calculators.cc

@@ -0,0 +1,403 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Basic Calculators that operate on TimeSeries streams.
+#include "mediapipe/calculators/audio/basic_time_series_calculators.h"
+
+#include <cmath>
+#include <memory>
+
+#include "Eigen/Core"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+namespace {
+static bool SafeMultiply(int x, int y, int* result) {
+  static_assert(sizeof(int64) >= 2 * sizeof(int),
+                "Unable to detect overflow after multiplication");
+  const int64 big = static_cast<int64>(x) * static_cast<int64>(y);
+  if (big > static_cast<int64>(INT_MIN) && big < static_cast<int64>(INT_MAX)) {
+    if (result != nullptr) *result = static_cast<int>(big);
+    return true;
+  } else {
+    return false;
+  }
+}
+}  // namespace
+
+::mediapipe::Status BasicTimeSeriesCalculatorBase::GetContract(
+    CalculatorContract* cc) {
+  cc->Inputs().Index(0).Set<Matrix>(
+      // Input stream with TimeSeriesHeader.
+  );
+  cc->Outputs().Index(0).Set<Matrix>(
+      // Output stream with TimeSeriesHeader.
+  );
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
+  TimeSeriesHeader input_header;
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+      cc->Inputs().Index(0).Header(), &input_header));
+
+  auto output_header = new TimeSeriesHeader(input_header);
+  MP_RETURN_IF_ERROR(MutateHeader(output_header));
+  cc->Outputs().Index(0).SetHeader(Adopt(output_header));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
+    CalculatorContext* cc) {
+  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+  MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
+      input, cc->Inputs().Index(0).Header().Get<TimeSeriesHeader>()));
+
+  std::unique_ptr<Matrix> output(new Matrix(ProcessMatrix(input)));
+  MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
+      *output, cc->Outputs().Index(0).Header().Get<TimeSeriesHeader>()));
+
+  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BasicTimeSeriesCalculatorBase::MutateHeader(
+    TimeSeriesHeader* output_header) {
+  return ::mediapipe::OkStatus();
+}
+
+// Calculator to sum an input time series across channels.  This is
+// useful for e.g. computing 'summary SAI' pitchogram features.
+//
+// Options proto: None.
+class SumTimeSeriesAcrossChannelsCalculator
+    : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_channels(1);
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.colwise().sum();
+  }
+};
+REGISTER_CALCULATOR(SumTimeSeriesAcrossChannelsCalculator);
+
+// Calculator to average an input time series across channels.  This is
+// useful for e.g. converting stereo or multi-channel files to mono.
+//
+// Options proto: None.
+class AverageTimeSeriesAcrossChannelsCalculator
+    : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_channels(1);
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.colwise().mean();
+  }
+};
+REGISTER_CALCULATOR(AverageTimeSeriesAcrossChannelsCalculator);
+
+// Calculator to convert a (temporal) summary SAI stream (a single-channel
+// stream output by SumTimeSeriesAcrossChannelsCalculator) into pitchogram
+// frames by transposing the input packets, swapping the time and channel axes.
+//
+// Options proto: None.
+class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    if (output_header->num_channels() != 1) {
+      return tool::StatusInvalid(
+          absl::StrCat("Expected single-channel input, got ",
+                       output_header->num_channels()));
+    }
+    output_header->set_num_channels(output_header->num_samples());
+    output_header->set_num_samples(1);
+    output_header->set_sample_rate(output_header->packet_rate());
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.transpose();
+  }
+};
+REGISTER_CALCULATOR(SummarySaiToPitchogramCalculator);
+
+// Calculator to reverse the order of channels in TimeSeries packets.
+// This is useful for e.g. interfacing with the speech pipeline which uses the
+// opposite convention to the hearing filterbanks.
+//
+// Options proto: None.
+class ReverseChannelOrderCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.colwise().reverse();
+  }
+};
+REGISTER_CALCULATOR(ReverseChannelOrderCalculator);
+
+// Calculator to flatten all samples in a TimeSeries packet down into
+// a single 'sample' vector.  This is useful for e.g. stacking several
+// frames of features into a single feature vector.
+//
+// Options proto: None.
+class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    const int num_input_channels = output_header->num_channels();
+    const int num_input_samples = output_header->num_samples();
+    RET_CHECK(num_input_channels >= 0)
+        << "FlattenPacketCalculator: num_input_channels < 0";
+    RET_CHECK(num_input_samples >= 0)
+        << "FlattenPacketCalculator: num_input_samples < 0";
+    int output_num_channels;
+    RET_CHECK(SafeMultiply(num_input_channels, num_input_samples,
+                           &output_num_channels))
+        << "FlattenPacketCalculator: Multiplication failed.";
+    output_header->set_num_channels(output_num_channels);
+    output_header->set_num_samples(1);
+    output_header->set_sample_rate(output_header->packet_rate());
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    // Flatten by interleaving channels so that full samples are
+    // stacked on top of each other instead of interleaving samples
+    // from the same channel.
+    Matrix output(input_matrix.size(), 1);
+    for (int sample = 0; sample < input_matrix.cols(); ++sample) {
+      output.middleRows(sample * input_matrix.rows(), input_matrix.rows()) =
+          input_matrix.col(sample);
+    }
+    return output;
+  }
+};
+REGISTER_CALCULATOR(FlattenPacketCalculator);
+
+// Calculator to subtract the within-packet mean for each channel from each
+// corresponding channel.
+//
+// Options proto: None.
+class SubtractMeanCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    Matrix mean = input_matrix.rowwise().mean();
+    return input_matrix - mean.replicate(1, input_matrix.cols());
+  }
+};
+REGISTER_CALCULATOR(SubtractMeanCalculator);
+
+// Calculator to subtract the mean over all values (across all times and
+// channels) in a Packet from the values in that Packet.
+//
+// Options proto: None.
+class SubtractMeanAcrossChannelsCalculator
+    : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    auto mean = input_matrix.mean();
+    return (input_matrix.array() - mean).matrix();
+  }
+};
+REGISTER_CALCULATOR(SubtractMeanAcrossChannelsCalculator);
+
+// Calculator to divide all values in a Packet by the average value across all
+// times and channels in the packet. This is useful for normalizing
+// nonnegative quantities like power, but might cause unexpected results if used
+// with Packets that can contain negative numbers.
+//
+// If mean is exactly zero, the output will be a matrix of all ones, because
+// that's what happens in other cases where all values are equal.
+//
+// Options proto: None.
+class DivideByMeanAcrossChannelsCalculator
+    : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    auto mean = input_matrix.mean();
+
+    if (mean != 0) {
+      return input_matrix / mean;
+
+      // When used with nonnegative matrices, the mean will only be zero if the
+      // entire matrix is exactly zero. If mean is exactly zero, the output will
+      // be a matrix of all ones, because that's what happens in other cases
+      // where
+      // all values are equal.
+    } else {
+      return Matrix::Ones(input_matrix.rows(), input_matrix.cols());
+    }
+  }
+};
+REGISTER_CALCULATOR(DivideByMeanAcrossChannelsCalculator);
+
+// Calculator to calculate the mean for each channel.
+//
+// Options proto: None.
+class MeanCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_samples(1);
+    output_header->set_sample_rate(output_header->packet_rate());
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.rowwise().mean();
+  }
+};
+REGISTER_CALCULATOR(MeanCalculator);
+
+// Calculator to calculate the uncorrected sample standard deviation in each
+// channel, independently for each Packet.  I.e. divide by the number of samples
+// in the Packet, not (<number of samples> - 1).
+//
+// Options proto: None.
+class StandardDeviationCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_samples(1);
+    output_header->set_sample_rate(output_header->packet_rate());
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    Eigen::VectorXf mean = input_matrix.rowwise().mean();
+    return (input_matrix.colwise() - mean).rowwise().norm() /
+           sqrt(input_matrix.cols());
+  }
+};
+REGISTER_CALCULATOR(StandardDeviationCalculator);
+
+// Calculator to calculate the covariance matrix. If the input matrix
+// has N channels, the output matrix will be an N by N symmetric
+// matrix.
+//
+// Options proto: None.
+class CovarianceCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_samples(output_header->num_channels());
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    auto mean = input_matrix.rowwise().mean();
+    auto zero_mean_input =
+        input_matrix - mean.replicate(1, input_matrix.cols());
+    return (zero_mean_input * zero_mean_input.transpose()) /
+           input_matrix.cols();
+  }
+};
+REGISTER_CALCULATOR(CovarianceCalculator);
+
+// Calculator to get the per column L2 norm of an input time series.
+//
+// Options proto: None.
+class L2NormCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    output_header->set_num_channels(1);
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.colwise().norm();
+  }
+};
+REGISTER_CALCULATOR(L2NormCalculator);
+
+// Calculator to convert each column of a matrix to a unit vector.
+//
+// Options proto: None.
+class L2NormalizeColumnCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.colwise().normalized();
+  }
+};
+REGISTER_CALCULATOR(L2NormalizeColumnCalculator);
+
+// Calculator to apply L2 normalization to the input matrix.
+//
+// Returns the matrix as is if the RMS is <= 1E-8.
+// Options proto: None.
+class L2NormalizeCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    constexpr double kEpsilon = 1e-8;
+    double rms = std::sqrt(input_matrix.array().square().mean());
+    if (rms <= kEpsilon) {
+      return input_matrix;
+    }
+    return input_matrix / rms;
+  }
+};
+REGISTER_CALCULATOR(L2NormalizeCalculator);
+
+// Calculator to apply Peak normalization to the input matrix.
+//
+// Returns the matrix as is if the peak is <= 1E-8.
+// Options proto: None.
+class PeakNormalizeCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    constexpr double kEpsilon = 1e-8;
+    double max_pcm = input_matrix.cwiseAbs().maxCoeff();
+    if (max_pcm <= kEpsilon) {
+      return input_matrix;
+    }
+    return input_matrix / max_pcm;
+  }
+};
+REGISTER_CALCULATOR(PeakNormalizeCalculator);
+
+// Calculator to compute the elementwise square of an input time series.
+//
+// Options proto: None.
+class ElementwiseSquareCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.array().square();
+  }
+};
+REGISTER_CALCULATOR(ElementwiseSquareCalculator);
+
+// Calculator that outputs first floor(num_samples / 2) of the samples.
+//
+// Options proto: None.
+class FirstHalfSlicerCalculator : public BasicTimeSeriesCalculatorBase {
+ protected:
+  ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header) final {
+    const int num_input_samples = output_header->num_samples();
+    RET_CHECK(num_input_samples >= 0)
+        << "FirstHalfSlicerCalculator: num_input_samples < 0";
+    output_header->set_num_samples(num_input_samples / 2);
+    return ::mediapipe::OkStatus();
+  }
+
+  Matrix ProcessMatrix(const Matrix& input_matrix) final {
+    return input_matrix.block(0, 0, input_matrix.rows(),
+                              input_matrix.cols() / 2);
+  }
+};
+REGISTER_CALCULATOR(FirstHalfSlicerCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/basic_time_series_calculators.h

@@ -0,0 +1,48 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Abstract base class for basic MediaPipe calculators that operate on
+// TimeSeries streams and don't require any Options protos.
+// Subclasses must override ProcessMatrix, and optionally
+// MutateHeader.
+
+#ifndef MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_
+#define MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+
+namespace mediapipe {
+
+class BasicTimeSeriesCalculatorBase : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ protected:
+  // Open() calls this method to mutate the output stream header.  The input
+  // to this function will contain a copy of the input stream header, so
+  // subclasses that do not need to mutate the header do not need to override
+  // it.
+  virtual ::mediapipe::Status MutateHeader(TimeSeriesHeader* output_header);
+
+  // Process() calls this method on each packet to compute the output matrix.
+  virtual Matrix ProcessMatrix(const Matrix& input_matrix) = 0;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_

mediapipe/calculators/audio/basic_time_series_calculators_test.cc

@@ -0,0 +1,515 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Eigen/Core"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+class SumTimeSeriesAcrossChannelsCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "SumTimeSeriesAcrossChannelsCalculator";
+  }
+};
+
+TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, IsNoOpOnSingleChannelInputs) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 1  num_samples: 5");
+  const Matrix input =
+      Matrix::Random(header.num_channels(), header.num_samples());
+
+  Test(header, {input}, header, {input});
+}
+
+TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 3  num_samples: 5");
+  TimeSeriesHeader output_header(header);
+  output_header.set_num_channels(1);
+
+  Test(header,
+       {Matrix::Constant(header.num_channels(), header.num_samples(), 1)},
+       output_header,
+       {Matrix::Constant(1, header.num_samples(), header.num_channels())});
+}
+
+TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, MultiplePackets) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 3  num_samples: 5");
+  Matrix in(header.num_channels(), header.num_samples());
+  in << 10, -1, -1, 0, 0, 20, -2, 0, 1, 0, 30, -3, 1, 0, 12;
+
+  TimeSeriesHeader output_header(header);
+  output_header.set_num_channels(1);
+  Matrix out(1, header.num_samples());
+  out << 60, -6, 0, 1, 12;
+
+  Test(header, {in, 2 * in, in + Matrix::Constant(in.rows(), in.cols(), 3.5f)},
+       output_header,
+       {out, 2 * out,
+        out + Matrix::Constant(out.rows(), out.cols(),
+                               3.5 * header.num_channels())});
+}
+
+class AverageTimeSeriesAcrossChannelsCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "AverageTimeSeriesAcrossChannelsCalculator";
+  }
+};
+
+TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest,
+       IsNoOpOnSingleChannelInputs) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 1  num_samples: 5");
+  const Matrix input =
+      Matrix::Random(header.num_channels(), header.num_samples());
+
+  Test(header, {input}, header, {input});
+}
+
+TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 3  num_samples: 5");
+  TimeSeriesHeader output_header(header);
+  output_header.set_num_channels(1);
+
+  Matrix input =
+      Matrix::Constant(header.num_channels(), header.num_samples(), 0.0);
+  input.row(0) = Matrix::Constant(1, header.num_samples(), 1.0);
+
+  Test(
+      header, {input}, output_header,
+      {Matrix::Constant(1, header.num_samples(), 1.0 / header.num_channels())});
+}
+
+class SummarySaiToPitchogramCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "SummarySaiToPitchogramCalculator";
+  }
+};
+
+TEST_F(SummarySaiToPitchogramCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 1  num_samples: 3");
+  Matrix input(1, input_header.num_samples());
+  input << 3, -9, 4;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 5.0  packet_rate: 5.0  num_channels: 3  num_samples: 1");
+  Matrix output(input_header.num_samples(), 1);
+  output << 3, -9, 4;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class ReverseChannelOrderCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "ReverseChannelOrderCalculator"; }
+};
+
+TEST_F(ReverseChannelOrderCalculatorTest, IsNoOpOnSingleChannelInputs) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 1  num_samples: 5");
+  const Matrix input =
+      Matrix::Random(header.num_channels(), header.num_samples());
+
+  Test(header, {input}, header, {input});
+}
+
+TEST_F(ReverseChannelOrderCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  num_channels: 5  num_samples: 2");
+  Matrix input(header.num_channels(), header.num_samples());
+  input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
+  Matrix output(header.num_channels(), header.num_samples());
+  output.transpose() << 5, 4, 3, 2, 1, -5, -4, -3, -2, -1;
+
+  Test(header, {input}, header, {output});
+}
+
+class FlattenPacketCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "FlattenPacketCalculator"; }
+};
+
+TEST_F(FlattenPacketCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
+  Matrix output(10, 1);
+  output << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 10.0  packet_rate: 10.0  num_channels: 10  num_samples: 1");
+  Test(input_header, {input}, output_header, {output});
+}
+
+class SubtractMeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "SubtractMeanCalculator"; }
+};
+
+TEST_F(SubtractMeanCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  Matrix output(input_header.num_channels(), input_header.num_samples());
+
+  // clang-format off
+  input.transpose() << 1,  0,  3, 0, 1,
+                      -1, -2, -3, 4, 7;
+  output.transpose() << 1,  1,  3, -2, -3,
+                       -1, -1, -3,  2,  3;
+  // clang-format on
+
+  const TimeSeriesHeader output_header = input_header;
+  Test(input_header, {input}, output_header, {output});
+}
+
+class SubtractMeanAcrossChannelsCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "SubtractMeanAcrossChannelsCalculator";
+  }
+};
+
+TEST_F(SubtractMeanAcrossChannelsCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_num_samples(2);
+
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+
+  // clang-format off
+  input.transpose() << 1.0, 2.0, 3.0,
+                       4.0, 5.0, 6.0;
+  output.transpose() << 1.0 - 3.5, 2.0 - 3.5, 3.0 - 3.5,
+                        4.0 - 3.5, 5.0 - 3.5, 6.0 - 3.5;
+  // clang-format on
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class DivideByMeanAcrossChannelsCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "DivideByMeanAcrossChannelsCalculator";
+  }
+};
+
+TEST_F(DivideByMeanAcrossChannelsCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0;
+
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_num_samples(2);
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output.transpose() << 1.0 / 3.5, 2.0 / 3.5, 3.0 / 3.5, 4.0 / 3.5, 5.0 / 3.5,
+      6.0 / 3.5;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+TEST_F(DivideByMeanAcrossChannelsCalculatorTest, ReturnsOneForZeroMean) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input.transpose() << -3.0, -2.0, -1.0, 1.0, 2.0, 3.0;
+
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_num_samples(2);
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output.transpose() << 1.0, 1.0, 1.0, 1.0, 1.0, 1.0;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class MeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "MeanCalculator"; }
+};
+
+TEST_F(MeanCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0;
+
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_num_samples(1);
+  output_header.set_sample_rate(10.0);
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << (1.0 + 4.0) / 2, (2.0 + 5.0) / 2, (3.0 + 6.0) / 2;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class StandardDeviationCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "StandardDeviationCalculator"; }
+};
+
+TEST_F(StandardDeviationCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input.transpose() << 0.0, 2.0, 3.0, 4.0, 5.0, 8.0;
+
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_sample_rate(10.0);
+  output_header.set_num_samples(1);
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << sqrt((pow(0.0 - 2.0, 2) + pow(4.0 - 2.0, 2)) / 2),
+      sqrt((pow(2.0 - 3.5, 2) + pow(5.0 - 3.5, 2)) / 2),
+      sqrt((pow(3.0 - 5.5, 2) + pow(8.0 - 5.5, 2)) / 2);
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class CovarianceCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "CovarianceCalculator"; }
+};
+
+TEST_F(CovarianceCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+
+  // We'll specify in transposed form so we can write one channel at a time.
+  input << 1.0, 3.0, 5.0, 9.0, -1.0, -3.0;
+
+  TimeSeriesHeader output_header(input_header);
+  output_header.set_num_samples(output_header.num_channels());
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << 1, 2, -1, 2, 4, -2, -1, -2, 1;
+  Test(input_header, {input}, output_header, {output});
+}
+
+class L2NormCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "L2NormCalculator"; }
+};
+
+TEST_F(L2NormCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 3, 5, 8, 4, 12, -15;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 1  num_samples: 3");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << 5, 13, 17;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class L2NormalizeColumnCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "L2NormalizeColumnCalculator"; }
+};
+
+TEST_F(L2NormalizeColumnCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+
+  // The values in output are column-wise L2 normalized
+  // e.g.
+  //    |a| -> |a/sqrt(a^2 + b^2)|
+  //    |b|    |b/sqrt(a^2 + b^2)|
+  output << 0.51449579000473022, 0.40613847970962524, 0.70710676908493042,
+      0.85749292373657227, 0.91381156444549561, 0.70710676908493042;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class L2NormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "L2NormalizeCalculator"; }
+};
+
+TEST_F(L2NormalizeCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+
+  // The values in output are L2 normalized
+  //    a -> a/sqrt(a^2 + b^2 + c^2 + ...) * sqrt(matrix.cols()*matrix.rows())
+  output << 0.45661166, 0.60881555, 1.21763109, 0.76101943, 1.36983498,
+      1.21763109;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+TEST_F(L2NormalizeCalculatorTest, UnitMatrixStaysUnchanged) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 3  num_samples: 5");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+      1.0, -1.0, 1.0;
+
+  Test(input_header, {input}, input_header, {input});
+}
+
+class PeakNormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "PeakNormalizeCalculator"; }
+};
+
+TEST_F(PeakNormalizeCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << 0.33333333, 0.44444444, 0.88888889, 0.55555556, 1.0, 0.88888889;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+TEST_F(PeakNormalizeCalculatorTest, UnitMatrixStaysUnchanged) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 3  num_samples: 5");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0,
+      1.0, -1.0, 1.0;
+
+  Test(input_header, {input}, input_header, {input});
+}
+
+class ElementwiseSquareCalculatorTest
+    : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "ElementwiseSquareCalculator"; }
+};
+
+TEST_F(ElementwiseSquareCalculatorTest, SinglePacket) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  input << 3, 5, 8, 4, 12, -15;
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 8000.0  packet_rate: 5.0  num_channels: 2  num_samples: 3");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output << 9, 25, 64, 16, 144, 225;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+class FirstHalfSlicerCalculatorTest : public BasicTimeSeriesCalculatorTestBase {
+ protected:
+  void SetUp() override { calculator_name_ = "FirstHalfSlicerCalculator"; }
+};
+
+TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketEvenNumSamples) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  // clang-format off
+  input.transpose() << 0, 1, 2, 3, 4,
+                       5, 6, 7, 8, 9;
+  // clang-format on
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 1");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output.transpose() << 0, 1, 2, 3, 4;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketOddNumSamples) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 3");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  // clang-format off
+  input.transpose() << 0, 1, 2, 3, 4,
+                       5, 6, 7, 8, 9,
+                       0, 0, 0, 0, 0;
+  // clang-format on
+
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 1");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output.transpose() << 0, 1, 2, 3, 4;
+
+  Test(input_header, {input}, output_header, {output});
+}
+
+TEST_F(FirstHalfSlicerCalculatorTest, MultiplePackets) {
+  const TimeSeriesHeader input_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 2");
+  Matrix input(input_header.num_channels(), input_header.num_samples());
+  // clang-format off
+  input.transpose() << 0, 1, 2, 3, 4,
+                       5, 6, 7, 8, 9;
+  // clang-format on
+  const TimeSeriesHeader output_header = ParseTextProtoOrDie<TimeSeriesHeader>(
+      "sample_rate: 20.0  packet_rate: 10.0  num_channels: 5  num_samples: 1");
+  Matrix output(output_header.num_channels(), output_header.num_samples());
+  output.transpose() << 0, 1, 2, 3, 4;
+
+  Test(input_header,
+       {input, 2 * input,
+        input + Matrix::Constant(input.rows(), input.cols(), 3.5f)},
+       output_header,
+       {output, 2 * output,
+        output + Matrix::Constant(output.rows(), output.cols(), 3.5f)});
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/mfcc_mel_calculators.cc

@@ -0,0 +1,275 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// MediaPipe Calculator wrapper around audio/dsp/mfcc/
+// classes MelFilterbank (magnitude spectrograms warped to the Mel
+// approximation of the auditory frequency scale) and Mfcc (Mel Frequency
+// Cepstral Coefficients, the decorrelated transform of log-Mel-spectrum
+// commonly used as acoustic features in speech and other audio tasks.
+// Both calculators expect as input the SQUARED_MAGNITUDE-domain outputs
+// from the MediaPipe SpectrogramCalculator object.
+#include <memory>
+#include <vector>
+
+#include "Eigen/Core"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "audio/dsp/mfcc/mel_filterbank.h"
+#include "audio/dsp/mfcc/mfcc.h"
+#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+namespace {
+
+// Portable version of TimeSeriesHeader's DebugString.
+std::string PortableDebugString(const TimeSeriesHeader& header) {
+  std::string unsubstituted_header_debug_str = R"(
+    sample_rate: $0
+    num_channels: $1
+    num_samples: $2
+    packet_rate: $3
+    audio_sample_rate: $4
+  )";
+  return absl::Substitute(unsubstituted_header_debug_str, header.sample_rate(),
+                          header.num_channels(), header.num_samples(),
+                          header.packet_rate(), header.audio_sample_rate());
+}
+
+}  // namespace
+
+// Abstract base class for Calculators that transform feature vectors on a
+// frame-by-frame basis.
+// Subclasses must override pure virtual methods ConfigureTransform and
+// TransformFrame.
+// Input and output MediaPipe packets are matrices with one column per frame,
+// and one row per feature dimension.  Each input packet results in an
+// output packet with the same number of columns (but differing numbers of
+// rows corresponding to the new feature space).
+class FramewiseTransformCalculatorBase : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Sequence of Matrices, each column describing a particular time frame,
+        // each row a feature dimension, with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Sequence of Matrices, each column describing a particular time frame,
+        // each row a feature dimension, with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+  int num_output_channels(void) { return num_output_channels_; }
+
+  void set_num_output_channels(int num_output_channels) {
+    num_output_channels_ = num_output_channels;
+  }
+
+ private:
+  // Takes header and options, and sets up state including calling
+  // set_num_output_channels() on the base object.
+  virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                                 CalculatorContext* cc) = 0;
+
+  // Takes a vector<double> corresponding to an input frame, and
+  // perform the specific transformation to produce an output frame.
+  virtual void TransformFrame(const std::vector<double>& input,
+                              std::vector<double>* output) const = 0;
+
+ private:
+  int num_output_channels_;
+};
+
+::mediapipe::Status FramewiseTransformCalculatorBase::Open(
+    CalculatorContext* cc) {
+  TimeSeriesHeader input_header;
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+      cc->Inputs().Index(0).Header(), &input_header));
+
+  ::mediapipe::Status status = ConfigureTransform(input_header, cc);
+
+  auto output_header = new TimeSeriesHeader(input_header);
+  output_header->set_num_channels(num_output_channels_);
+  cc->Outputs().Index(0).SetHeader(Adopt(output_header));
+
+  return status;
+}
+
+::mediapipe::Status FramewiseTransformCalculatorBase::Process(
+    CalculatorContext* cc) {
+  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+  const int num_frames = input.cols();
+  std::unique_ptr<Matrix> output(new Matrix(num_output_channels_, num_frames));
+  // The main work here is converting each column of the float Matrix
+  // into a vector of doubles, which is what our target functions from
+  // dsp_core consume, and doing the reverse with their output.
+  std::vector<double> input_frame(input.rows());
+  std::vector<double> output_frame(num_output_channels_);
+
+  for (int frame = 0; frame < num_frames; ++frame) {
+    // Copy input from Eigen::Matrix column to vector<float>.
+    Eigen::Map<Eigen::MatrixXd> input_frame_map(&input_frame[0],
+                                                input_frame.size(), 1);
+    input_frame_map = input.col(frame).cast<double>();
+
+    // Perform the actual transformation.
+    TransformFrame(input_frame, &output_frame);
+
+    // Copy output from vector<float> to Eigen::Vector.
+    CHECK_EQ(output_frame.size(), num_output_channels_);
+    Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
+                                                       output_frame.size(), 1);
+    output->col(frame) = output_frame_map.cast<float>();
+  }
+  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+
+  return ::mediapipe::OkStatus();
+}
+
+// Calculator wrapper around the dsp/mfcc/mfcc.cc routine.
+// Take frames of squared-magnitude spectra from the SpectrogramCalculator
+// and convert them into Mel Frequency Cepstral Coefficients.
+//
+// Example config:
+// node {
+//   calculator: "MfccCalculator"
+//   input_stream: "spectrogram_frames_stream"
+//   output_stream: "mfcc_frames_stream"
+//   options {
+//     [mediapipe.MfccCalculatorOptions.ext] {
+//       mel_spectrum_params {
+//         channel_count: 20
+//         min_frequency_hertz: 125.0
+//         max_frequency_hertz: 3800.0
+//       }
+//       mfcc_count: 13
+//     }
+//   }
+// }
+class MfccCalculator : public FramewiseTransformCalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    return FramewiseTransformCalculatorBase::GetContract(cc);
+  }
+
+ private:
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
+    MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
+    mfcc_.reset(new audio_dsp::Mfcc());
+    int input_length = header.num_channels();
+    // Set up the parameters to the Mfcc object.
+    set_num_output_channels(mfcc_options.mfcc_count());
+    mfcc_->set_dct_coefficient_count(num_output_channels());
+    mfcc_->set_upper_frequency_limit(
+        mfcc_options.mel_spectrum_params().max_frequency_hertz());
+    mfcc_->set_lower_frequency_limit(
+        mfcc_options.mel_spectrum_params().min_frequency_hertz());
+    mfcc_->set_filterbank_channel_count(
+        mfcc_options.mel_spectrum_params().channel_count());
+    // An upstream calculator (such as SpectrogramCalculator) must store
+    // the sample rate of its input audio waveform in the TimeSeries Header.
+    // audio_dsp::MelFilterBank needs to know this to
+    // correctly interpret the spectrogram bins.
+    if (!header.has_audio_sample_rate()) {
+      return ::mediapipe::InvalidArgumentError(
+          absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
+                       PortableDebugString(header)));
+    }
+    // Now we can initialize the Mfcc object.
+    bool initialized =
+        mfcc_->Initialize(input_length, header.audio_sample_rate());
+
+    if (initialized) {
+      return ::mediapipe::OkStatus();
+    } else {
+      return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
+                                 "Mfcc::Initialize returned uninitialized");
+    }
+  }
+
+  void TransformFrame(const std::vector<double>& input,
+                      std::vector<double>* output) const override {
+    mfcc_->Compute(input, output);
+  }
+
+ private:
+  std::unique_ptr<audio_dsp::Mfcc> mfcc_;
+};
+REGISTER_CALCULATOR(MfccCalculator);
+
+// Calculator wrapper around the dsp/mfcc/mel_filterbank.cc routine.
+// Take frames of squared-magnitude spectra from the SpectrogramCalculator
+// and convert them into Mel-warped (linear-magnitude) spectra.
+// Note: This code computes a mel-frequency filterbank, using a simple
+// algorithm that gives bad results (some mel channels that are always zero)
+// if you ask for too many channels.
+class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    return FramewiseTransformCalculatorBase::GetContract(cc);
+  }
+
+ private:
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
+    MelSpectrumCalculatorOptions mel_spectrum_options =
+        cc->Options<MelSpectrumCalculatorOptions>();
+    mel_filterbank_.reset(new audio_dsp::MelFilterbank());
+    int input_length = header.num_channels();
+    set_num_output_channels(mel_spectrum_options.channel_count());
+    // An upstream calculator (such as SpectrogramCalculator) must store
+    // the sample rate of its input audio waveform in the TimeSeries Header.
+    // audio_dsp::MelFilterBank needs to know this to
+    // correctly interpret the spectrogram bins.
+    if (!header.has_audio_sample_rate()) {
+      return ::mediapipe::InvalidArgumentError(
+          absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ",
+                       PortableDebugString(header)));
+    }
+    bool initialized = mel_filterbank_->Initialize(
+        input_length, header.audio_sample_rate(), num_output_channels(),
+        mel_spectrum_options.min_frequency_hertz(),
+        mel_spectrum_options.max_frequency_hertz());
+
+    if (initialized) {
+      return ::mediapipe::OkStatus();
+    } else {
+      return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
+                                 "mfcc::Initialize returned uninitialized");
+    }
+  }
+
+  void TransformFrame(const std::vector<double>& input,
+                      std::vector<double>* output) const override {
+    mel_filterbank_->Compute(input, output);
+  }
+
+ private:
+  std::unique_ptr<audio_dsp::MelFilterbank> mel_filterbank_;
+};
+REGISTER_CALCULATOR(MelSpectrumCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/mfcc_mel_calculators.proto

@@ -0,0 +1,50 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message MelSpectrumCalculatorOptions {
+  extend CalculatorOptions {
+    optional MelSpectrumCalculatorOptions ext = 78581812;
+  }
+  // The fields are to populate the config parameters in
+  // audio/dsp/mfcc/mel_filterbank.h
+  // but the names are chose to mirror
+  // audio/hearing/filterbanks/cochlea_gammatone_filterbank.proto
+  // and the default values match those in
+  // speech/greco3/frontend/filter_bank.proto .
+
+  // Total number of frequency bands to use.
+  optional int32 channel_count = 1 [default = 20];
+  // Lower edge of lowest triangular Mel band.
+  optional float min_frequency_hertz = 2 [default = 125.0];
+  // Upper edge of highest triangular Mel band.
+  optional float max_frequency_hertz = 3 [default = 3800.0];
+}
+
+message MfccCalculatorOptions {
+  extend CalculatorOptions {
+    optional MfccCalculatorOptions ext = 78450441;
+  }
+
+  // Specification of the underlying mel filterbank.
+  optional MelSpectrumCalculatorOptions mel_spectrum_params = 1;
+
+  // How many MFCC coefficients to emit.
+  optional uint32 mfcc_count = 2 [default = 13];
+}

mediapipe/calculators/audio/mfcc_mel_calculators_test.cc

@@ -0,0 +1,149 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <vector>
+
+#include "Eigen/Core"
+#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+// Use a sample rate that is unlikely to be a default somewhere.
+const float kAudioSampleRate = 8800.0;
+
+template <typename OptionsType, const char* CalculatorName>
+class FramewiseTransformCalculatorTest
+    : public TimeSeriesCalculatorTest<OptionsType> {
+ protected:
+  void SetUp() override {
+    this->calculator_name_ = CalculatorName;
+    this->num_input_channels_ = 129;
+    // This is the frame rate coming out of the SpectrogramCalculator.
+    this->input_sample_rate_ = 100.0;
+  }
+
+  // Returns the number of samples per packet.
+  int GenerateRandomNonnegInputStream(int num_packets) {
+    const double kSecondsPerPacket = 0.2;
+    const int num_samples_per_packet =
+        kSecondsPerPacket * this->input_sample_rate_;
+    for (int i = 0; i < num_packets; ++i) {
+      const int timestamp =
+          i * kSecondsPerPacket * Timestamp::kTimestampUnitsPerSecond;
+      // Mfcc, MelSpectrum expect squared-magnitude inputs, so make
+      // sure the input data has no negative values.
+      Matrix* sqdata = this->NewRandomMatrix(this->num_input_channels_,
+                                             num_samples_per_packet);
+      *sqdata = sqdata->array().square();
+      this->AppendInputPacket(sqdata, timestamp);
+    }
+    return num_samples_per_packet;
+  }
+
+  void CheckOutputPacketMetadata(int expected_num_channels,
+                                 int expected_num_samples_per_packet) {
+    int expected_timestamp = 0;
+    for (const auto& packet : this->output().packets) {
+      EXPECT_EQ(expected_timestamp, packet.Timestamp().Value());
+      expected_timestamp += expected_num_samples_per_packet /
+                            this->input_sample_rate_ *
+                            Timestamp::kTimestampUnitsPerSecond;
+
+      const Matrix& output_matrix = packet.template Get<Matrix>();
+
+      EXPECT_EQ(output_matrix.rows(), expected_num_channels);
+      EXPECT_EQ(output_matrix.cols(), expected_num_samples_per_packet);
+    }
+  }
+
+  void SetupGraphAndHeader() {
+    this->InitializeGraph();
+    this->FillInputHeader();
+  }
+
+  // Argument is the expected number of dimensions (channels, columns) in
+  // the output data from the Calculator under test, which the test should
+  // know.
+  void SetupRandomInputPackets() {
+    constexpr int kNumPackets = 5;
+    num_samples_per_packet_ = GenerateRandomNonnegInputStream(kNumPackets);
+  }
+
+  ::mediapipe::Status Run() { return this->RunGraph(); }
+
+  void CheckResults(int expected_num_channels) {
+    const auto& output_header =
+        this->output().header.template Get<TimeSeriesHeader>();
+    EXPECT_EQ(this->input_sample_rate_, output_header.sample_rate());
+    CheckOutputPacketMetadata(expected_num_channels, num_samples_per_packet_);
+
+    // Sanity check that output packets have non-zero energy.
+    for (const auto& packet : this->output().packets) {
+      const Matrix& data = packet.template Get<Matrix>();
+      EXPECT_GT(data.squaredNorm(), 0);
+    }
+  }
+
+  // Allows SetupRandomInputPackets() to inform CheckResults() about how
+  // big the packets are supposed to be.
+  int num_samples_per_packet_;
+};
+
+constexpr char kMfccCalculator[] = "MfccCalculator";
+typedef FramewiseTransformCalculatorTest<MfccCalculatorOptions, kMfccCalculator>
+    MfccCalculatorTest;
+TEST_F(MfccCalculatorTest, AudioSampleRateFromInputHeader) {
+  audio_sample_rate_ = kAudioSampleRate;
+  SetupGraphAndHeader();
+  SetupRandomInputPackets();
+
+  MP_EXPECT_OK(Run());
+
+  CheckResults(options_.mfcc_count());
+}
+TEST_F(MfccCalculatorTest, NoAudioSampleRate) {
+  // Leave audio_sample_rate_ == kUnset, so it is not present in the
+  // input TimeSeriesHeader; expect failure.
+  SetupGraphAndHeader();
+  SetupRandomInputPackets();
+
+  EXPECT_FALSE(Run().ok());
+}
+
+constexpr char kMelSpectrumCalculator[] = "MelSpectrumCalculator";
+typedef FramewiseTransformCalculatorTest<MelSpectrumCalculatorOptions,
+                                         kMelSpectrumCalculator>
+    MelSpectrumCalculatorTest;
+TEST_F(MelSpectrumCalculatorTest, AudioSampleRateFromInputHeader) {
+  audio_sample_rate_ = kAudioSampleRate;
+  SetupGraphAndHeader();
+  SetupRandomInputPackets();
+
+  MP_EXPECT_OK(Run());
+
+  CheckResults(options_.channel_count());
+}
+TEST_F(MelSpectrumCalculatorTest, NoAudioSampleRate) {
+  // Leave audio_sample_rate_ == kUnset, so it is not present in the
+  // input TimeSeriesHeader; expect failure.
+  SetupGraphAndHeader();
+  SetupRandomInputPackets();
+
+  EXPECT_FALSE(Run().ok());
+}
+}  // namespace mediapipe

mediapipe/calculators/audio/rational_factor_resample_calculator.cc

@@ -0,0 +1,197 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines RationalFactorResampleCalculator.
+
+#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
+
+#include "audio/dsp/resampler_rational_factor.h"
+
+using audio_dsp::DefaultResamplingKernel;
+using audio_dsp::RationalFactorResampler;
+using audio_dsp::Resampler;
+
+namespace mediapipe {
+::mediapipe::Status RationalFactorResampleCalculator::Process(
+    CalculatorContext* cc) {
+  return ProcessInternal(cc->Inputs().Index(0).Get<Matrix>(), false, cc);
+}
+
+::mediapipe::Status RationalFactorResampleCalculator::Close(
+    CalculatorContext* cc) {
+  if (initial_timestamp_ == Timestamp::Unstarted()) {
+    return ::mediapipe::OkStatus();
+  }
+  Matrix empty_input_frame(num_channels_, 0);
+  return ProcessInternal(empty_input_frame, true, cc);
+}
+
+namespace {
+void CopyChannelToVector(const Matrix& matrix, int channel,
+                         std::vector<float>* vec) {
+  vec->clear();
+  vec->reserve(matrix.cols());
+  for (int sample = 0; sample < matrix.cols(); ++sample) {
+    vec->push_back(matrix(channel, sample));
+  }
+}
+
+void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
+                         int channel) {
+  if (matrix->cols() == 0) {
+    matrix->resize(matrix->rows(), vec.size());
+  } else {
+    CHECK_EQ(vec.size(), matrix->cols());
+    CHECK_LT(channel, matrix->rows());
+  }
+  for (int sample = 0; sample < matrix->cols(); ++sample) {
+    (*matrix)(channel, sample) = vec[sample];
+  }
+}
+
+}  // namespace
+
+::mediapipe::Status RationalFactorResampleCalculator::Open(
+    CalculatorContext* cc) {
+  RationalFactorResampleCalculatorOptions resample_options =
+      cc->Options<RationalFactorResampleCalculatorOptions>();
+
+  if (!resample_options.has_target_sample_rate()) {
+    return tool::StatusInvalid(
+        "resample_options doesn't have target_sample_rate.");
+  }
+  target_sample_rate_ = resample_options.target_sample_rate();
+
+  TimeSeriesHeader input_header;
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+      cc->Inputs().Index(0).Header(), &input_header));
+
+  source_sample_rate_ = input_header.sample_rate();
+  num_channels_ = input_header.num_channels();
+
+  // Don't create resamplers for pass-thru (sample rates are equal).
+  if (source_sample_rate_ != target_sample_rate_) {
+    resampler_.resize(num_channels_);
+    for (auto& r : resampler_) {
+      r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
+                               resample_options);
+      if (!r) {
+        LOG(ERROR) << "Failed to initialize resampler.";
+        return ::mediapipe::UnknownError("Failed to initialize resampler.");
+      }
+    }
+  }
+
+  TimeSeriesHeader* output_header = new TimeSeriesHeader(input_header);
+  output_header->set_sample_rate(target_sample_rate_);
+  // The resampler doesn't make guarantees about how many samples will
+  // be in each packet.
+  output_header->clear_packet_rate();
+  output_header->clear_num_samples();
+
+  cc->Outputs().Index(0).SetHeader(Adopt(output_header));
+  cumulative_output_samples_ = 0;
+  cumulative_input_samples_ = 0;
+  initial_timestamp_ = Timestamp::Unstarted();
+  check_inconsistent_timestamps_ =
+      resample_options.check_inconsistent_timestamps();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status RationalFactorResampleCalculator::ProcessInternal(
+    const Matrix& input_frame, bool should_flush, CalculatorContext* cc) {
+  if (initial_timestamp_ == Timestamp::Unstarted()) {
+    initial_timestamp_ = cc->InputTimestamp();
+  }
+
+  if (check_inconsistent_timestamps_) {
+    time_series_util::LogWarningIfTimestampIsInconsistent(
+        cc->InputTimestamp(), initial_timestamp_, cumulative_input_samples_,
+        source_sample_rate_);
+  }
+  Timestamp output_timestamp =
+      initial_timestamp_ + ((cumulative_output_samples_ / target_sample_rate_) *
+                            Timestamp::kTimestampUnitsPerSecond);
+
+  cumulative_input_samples_ += input_frame.cols();
+  std::unique_ptr<Matrix> output_frame(new Matrix(num_channels_, 0));
+  if (resampler_.empty()) {
+    // Sample rates were same for input and output; pass-thru.
+    *output_frame = input_frame;
+  } else {
+    if (!Resample(input_frame, output_frame.get(), should_flush)) {
+      return ::mediapipe::UnknownError("Resample() failed.");
+    }
+  }
+  cumulative_output_samples_ += output_frame->cols();
+
+  if (output_frame->cols() > 0) {
+    cc->Outputs().Index(0).Add(output_frame.release(), output_timestamp);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+bool RationalFactorResampleCalculator::Resample(const Matrix& input_frame,
+                                                Matrix* output_frame,
+                                                bool should_flush) {
+  std::vector<float> input_vector;
+  std::vector<float> output_vector;
+  for (int i = 0; i < input_frame.rows(); ++i) {
+    CopyChannelToVector(input_frame, i, &input_vector);
+    if (should_flush) {
+      resampler_[i]->Flush(&output_vector);
+    } else {
+      resampler_[i]->ProcessSamples(input_vector, &output_vector);
+    }
+    CopyVectorToChannel(output_vector, output_frame, i);
+  }
+  return true;
+}
+
+// static
+std::unique_ptr<Resampler<float>>
+RationalFactorResampleCalculator::ResamplerFromOptions(
+    const double source_sample_rate, const double target_sample_rate,
+    const RationalFactorResampleCalculatorOptions& options) {
+  std::unique_ptr<Resampler<float>> resampler;
+  const auto& rational_factor_options =
+      options.resampler_rational_factor_options();
+  std::unique_ptr<DefaultResamplingKernel> kernel;
+  if (rational_factor_options.has_radius() &&
+      rational_factor_options.has_cutoff() &&
+      rational_factor_options.has_kaiser_beta()) {
+    kernel = absl::make_unique<DefaultResamplingKernel>(
+        source_sample_rate, target_sample_rate,
+        rational_factor_options.radius(), rational_factor_options.cutoff(),
+        rational_factor_options.kaiser_beta());
+  } else {
+    kernel = absl::make_unique<DefaultResamplingKernel>(source_sample_rate,
+                                                        target_sample_rate);
+  }
+
+  // Set large enough so that the resampling factor between common sample
+  // rates (e.g. 8kHz, 16kHz, 22.05kHz, 32kHz, 44.1kHz, 48kHz) is exact, and
+  // that any factor is represented with error less than 0.025%.
+  const int kMaxDenominator = 2000;
+  resampler = absl::make_unique<RationalFactorResampler<float>>(
+      *kernel, kMaxDenominator);
+  if (resampler != nullptr && !resampler->Valid()) {
+    resampler = std::unique_ptr<Resampler<float>>();
+  }
+  return resampler;
+}
+
+REGISTER_CALCULATOR(RationalFactorResampleCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/rational_factor_resample_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "Eigen/Core"
+#include "absl/strings/str_cat.h"
+#include "audio/dsp/resampler.h"
+#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+// MediaPipe Calculator for resampling a (vector-valued)
+// input time series with a uniform sample rate.  The output
+// stream's sampling rate is specified by target_sample_rate in the
+// RationalFactorResampleCalculatorOptions.  The output time series may have
+// a varying number of samples per frame.
+class RationalFactorResampleCalculator : public CalculatorBase {
+ public:
+  struct TestAccess;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Single input stream with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Resampled stream with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+  // Returns FAIL if the input stream header is invalid or if the
+  // resampler cannot be initialized.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  // Resamples a packet of TimeSeries data.  Returns FAIL if the
+  // resampler state becomes inconsistent.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  // Flushes any remaining state.  Returns FAIL if the resampler state
+  // becomes inconsistent.
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ protected:
+  typedef audio_dsp::Resampler<float> ResamplerType;
+
+  // Returns a Resampler<float> implementation specified by the
+  // RationalFactorResampleCalculatorOptions proto. Returns null if the options
+  // specify an invalid resampler.
+  static std::unique_ptr<ResamplerType> ResamplerFromOptions(
+      const double source_sample_rate, const double target_sample_rate,
+      const RationalFactorResampleCalculatorOptions& options);
+
+  // Does Timestamp bookkeeping and resampling common to Process() and
+  // Close().  Returns FAIL if the resampler state becomes
+  // inconsistent.
+  ::mediapipe::Status ProcessInternal(const Matrix& input_frame,
+                                      bool should_flush, CalculatorContext* cc);
+
+  // Uses the internal resampler_ objects to actually resample each
+  // row of the input TimeSeries.  Returns false if the resampler
+  // state becomes inconsistent.
+  bool Resample(const Matrix& input_frame, Matrix* output_frame,
+                bool should_flush);
+
+  double source_sample_rate_;
+  double target_sample_rate_;
+  int64 cumulative_input_samples_;
+  int64 cumulative_output_samples_;
+  Timestamp initial_timestamp_;
+  bool check_inconsistent_timestamps_;
+  int num_channels_;
+  std::vector<std::unique_ptr<ResamplerType>> resampler_;
+};
+
+// Test-only access to RationalFactorResampleCalculator methods.
+struct RationalFactorResampleCalculator::TestAccess {
+  static std::unique_ptr<ResamplerType> ResamplerFromOptions(
+      const double source_sample_rate, const double target_sample_rate,
+      const RationalFactorResampleCalculatorOptions& options) {
+    return RationalFactorResampleCalculator::ResamplerFromOptions(
+        source_sample_rate, target_sample_rate, options);
+  }
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_

mediapipe/calculators/audio/rational_factor_resample_calculator.proto

@@ -0,0 +1,46 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message RationalFactorResampleCalculatorOptions {
+  extend CalculatorOptions {
+    optional RationalFactorResampleCalculatorOptions ext = 259760074;
+  }
+
+  // target_sample_rate is the sample rate, in Hertz, of the output
+  // stream.  Required.  Must be greater than 0.
+  optional double target_sample_rate = 1;
+
+  // Parameters for initializing the RationalFactorResampler. See
+  // RationalFactorResampler for more details.
+  message ResamplerRationalFactorOptions {
+    // Kernel radius in units of input samples.
+    optional double radius = 1;
+    // Anti-aliasing cutoff frequency in Hertz. A reasonable setting is
+    // 0.45 * min(input_sample_rate, output_sample_rate).
+    optional double cutoff = 2;
+    // The Kaiser beta parameter for the kernel window.
+    optional double kaiser_beta = 3 [default = 6.0];
+  }
+  optional ResamplerRationalFactorOptions resampler_rational_factor_options = 2;
+
+  // Set to false to disable checks for jitter in timestamp values. Useful with
+  // live audio input.
+  optional bool check_inconsistent_timestamps = 3 [default = true];
+}

mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc

@@ -0,0 +1,247 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
+
+#include <math.h>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "Eigen/Core"
+#include "audio/dsp/signal_vector_util.h"
+#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h"
+#include "mediapipe/framework//tool/validate_type.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+namespace {
+
+const int kInitialTimestampOffsetMilliseconds = 4;
+
+class RationalFactorResampleCalculatorTest
+    : public TimeSeriesCalculatorTest<RationalFactorResampleCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "RationalFactorResampleCalculator";
+    input_sample_rate_ = 4000.0;
+    num_input_channels_ = 3;
+  }
+
+  // Expects two vectors whose lengths are almost the same and whose
+  // elements are equal (for indices that are present in both).
+  //
+  // This is useful because the resampler doesn't make precise
+  // guarantees about its output size.
+  void ExpectVectorMostlyFloatEq(const std::vector<float>& expected,
+                                 const std::vector<float>& actual) {
+    // Lengths should be close, but don't have to be equal.
+    ASSERT_NEAR(expected.size(), actual.size(), 1);
+    for (int i = 0; i < std::min(expected.size(), actual.size()); ++i) {
+      EXPECT_FLOAT_EQ(expected[i], actual[i]) << " where i=" << i << ".";
+    }
+  }
+
+  // Returns a float value with the sample, channel, and timestamp
+  // separated by a few orders of magnitude, for easy parsing by
+  // humans.
+  double TestValue(int sample, int channel, int timestamp_in_microseconds) {
+    return timestamp_in_microseconds * 100.0 + sample + channel / 10.0;
+  }
+
+  // Caller takes ownership of the returned value.
+  Matrix* NewTestFrame(int num_channels, int num_samples, int timestamp) {
+    auto matrix = new Matrix(num_channels, num_samples);
+    for (int c = 0; c < num_channels; ++c) {
+      for (int i = 0; i < num_samples; ++i) {
+        (*matrix)(c, i) = TestValue(i, c, timestamp);
+      }
+    }
+    return matrix;
+  }
+
+  // Initializes and runs the test graph.
+  ::mediapipe::Status Run(double output_sample_rate) {
+    options_.set_target_sample_rate(output_sample_rate);
+    InitializeGraph();
+
+    FillInputHeader();
+    concatenated_input_samples_.resize(num_input_channels_, 0);
+    num_input_samples_ = 0;
+    for (int i = 0; i < 5; ++i) {
+      int packet_size = (i + 1) * 10;
+      int timestamp = kInitialTimestampOffsetMilliseconds +
+                      num_input_samples_ / input_sample_rate_ *
+                          Timestamp::kTimestampUnitsPerSecond;
+      Matrix* data_frame =
+          NewTestFrame(num_input_channels_, packet_size, timestamp);
+
+      // Keep a reference copy of the input.
+      //
+      // conservativeResize() is needed here to preserve the existing
+      // data.  Eigen's resize() resizes without preserving data.
+      concatenated_input_samples_.conservativeResize(
+          num_input_channels_, num_input_samples_ + packet_size);
+      concatenated_input_samples_.rightCols(packet_size) = *data_frame;
+      num_input_samples_ += packet_size;
+
+      AppendInputPacket(data_frame, timestamp);
+    }
+
+    return RunGraph();
+  }
+
+  void CheckOutputLength(double output_sample_rate) {
+    double factor = output_sample_rate / input_sample_rate_;
+
+    int num_output_samples = 0;
+    for (const Packet& packet : output().packets) {
+      num_output_samples += packet.Get<Matrix>().cols();
+    }
+
+    // The exact number of expected samples may vary based on the implementation
+    // of the resampler since the exact value is not an integer.
+    // TODO: Reduce this offset to + 1 once cl/185829520 is submitted.
+    const double expected_num_output_samples = num_input_samples_ * factor;
+    EXPECT_LE(ceil(expected_num_output_samples), num_output_samples);
+    EXPECT_GE(ceil(expected_num_output_samples) + 11, num_output_samples);
+  }
+
+  // Checks that output timestamps are consistent with the
+  // output_sample_rate and output packet sizes.
+  void CheckOutputPacketTimestamps(double output_sample_rate) {
+    int num_output_samples = 0;
+    for (const Packet& packet : output().packets) {
+      const int expected_timestamp = kInitialTimestampOffsetMilliseconds +
+                                     num_output_samples / output_sample_rate *
+                                         Timestamp::kTimestampUnitsPerSecond;
+      EXPECT_NEAR(expected_timestamp, packet.Timestamp().Value(), 1);
+      num_output_samples += packet.Get<Matrix>().cols();
+    }
+  }
+
+  // Checks that output values from the calculator (which resamples
+  // packet-by-packet) are consistent with resampling the entire
+  // signal at once.
+  void CheckOutputValues(double output_sample_rate) {
+    for (int i = 0; i < num_input_channels_; ++i) {
+      auto verification_resampler =
+          RationalFactorResampleCalculator::TestAccess::ResamplerFromOptions(
+              input_sample_rate_, output_sample_rate, options_);
+
+      std::vector<float> input_data;
+      for (int j = 0; j < num_input_samples_; ++j) {
+        input_data.push_back(concatenated_input_samples_(i, j));
+      }
+      std::vector<float> expected_resampled_data;
+      std::vector<float> temp;
+      verification_resampler->ProcessSamples(input_data, &temp);
+      audio_dsp::VectorAppend(&expected_resampled_data, temp);
+      verification_resampler->Flush(&temp);
+      audio_dsp::VectorAppend(&expected_resampled_data, temp);
+      std::vector<float> actual_resampled_data;
+      for (const Packet& packet : output().packets) {
+        Matrix output_frame_row = packet.Get<Matrix>().row(i);
+        actual_resampled_data.insert(
+            actual_resampled_data.end(), &output_frame_row(0),
+            &output_frame_row(0) + output_frame_row.cols());
+      }
+
+      ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data);
+    }
+  }
+
+  void CheckOutputHeaders(double output_sample_rate) {
+    const TimeSeriesHeader& output_header =
+        output().header.Get<TimeSeriesHeader>();
+    TimeSeriesHeader expected_header;
+    expected_header.set_sample_rate(output_sample_rate);
+    expected_header.set_num_channels(num_input_channels_);
+    EXPECT_THAT(output_header, mediapipe::EqualsProto(expected_header));
+  }
+
+  void CheckOutput(double output_sample_rate) {
+    CheckOutputLength(output_sample_rate);
+    CheckOutputPacketTimestamps(output_sample_rate);
+    CheckOutputValues(output_sample_rate);
+    CheckOutputHeaders(output_sample_rate);
+  }
+
+  void CheckOutputUnchanged() {
+    for (int i = 0; i < num_input_channels_; ++i) {
+      std::vector<float> expected_resampled_data;
+      for (int j = 0; j < num_input_samples_; ++j) {
+        expected_resampled_data.push_back(concatenated_input_samples_(i, j));
+      }
+      std::vector<float> actual_resampled_data;
+      for (const Packet& packet : output().packets) {
+        Matrix output_frame_row = packet.Get<Matrix>().row(i);
+        actual_resampled_data.insert(
+            actual_resampled_data.end(), &output_frame_row(0),
+            &output_frame_row(0) + output_frame_row.cols());
+      }
+      ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data);
+    }
+  }
+
+  int num_input_samples_;
+  Matrix concatenated_input_samples_;
+};
+
+TEST_F(RationalFactorResampleCalculatorTest, Upsample) {
+  const double kUpsampleRate = input_sample_rate_ * 1.9;
+  MP_ASSERT_OK(Run(kUpsampleRate));
+  CheckOutput(kUpsampleRate);
+}
+
+TEST_F(RationalFactorResampleCalculatorTest, Downsample) {
+  const double kDownsampleRate = input_sample_rate_ / 1.9;
+  MP_ASSERT_OK(Run(kDownsampleRate));
+  CheckOutput(kDownsampleRate);
+}
+
+TEST_F(RationalFactorResampleCalculatorTest, UsesRationalFactorResampler) {
+  const double kUpsampleRate = input_sample_rate_ * 2;
+  MP_ASSERT_OK(Run(kUpsampleRate));
+  CheckOutput(kUpsampleRate);
+}
+
+TEST_F(RationalFactorResampleCalculatorTest, PassthroughIfSampleRateUnchanged) {
+  const double kUpsampleRate = input_sample_rate_;
+  MP_ASSERT_OK(Run(kUpsampleRate));
+  CheckOutputUnchanged();
+}
+
+TEST_F(RationalFactorResampleCalculatorTest, FailsOnBadTargetRate) {
+  ASSERT_FALSE(Run(-999.9).ok());  // Invalid output sample rate.
+}
+
+TEST_F(RationalFactorResampleCalculatorTest, DoesNotDieOnEmptyInput) {
+  options_.set_target_sample_rate(input_sample_rate_);
+  InitializeGraph();
+  FillInputHeader();
+  MP_ASSERT_OK(RunGraph());
+  EXPECT_TRUE(output().packets.empty());
+}
+
+}  // anonymous namespace
+}  // namespace mediapipe

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -0,0 +1,436 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines SpectrogramCalculator.
+#include <math.h>
+
+#include <complex>
+#include <deque>
+#include <memory>
+#include <string>
+
+#include "Eigen/Core"
+#include "absl/strings/string_view.h"
+#include "audio/dsp/spectrogram/spectrogram.h"
+#include "audio/dsp/window_functions.h"
+#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/core_proto_inc.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/source_location.h"
+#include "mediapipe/framework/port/status_builder.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// MediaPipe Calculator for computing the "spectrogram" (short-time Fourier
+// transform squared-magnitude, by default) of a multichannel input
+// time series, including optionally overlapping frames.  Options are
+// specified in SpectrogramCalculatorOptions proto  (where names are chosen
+// to mirror TimeSeriesFramerCalculator):
+//
+// Result is a MatrixData record (for single channel input and when the
+// allow_multichannel_input flag is false), or a vector of MatrixData records,
+// one for each channel (when the allow_multichannel_input flag is set). The
+// rows of each spectrogram matrix correspond to the n_fft/2+1 unique complex
+// values, or squared/linear/dB magnitudes, depending on the output_type option.
+// Each input packet will result in zero or one output packets, each containing
+// one Matrix for each channel of the input, where each Matrix has one or more
+// columns of spectral values, one for each complete frame of input samples. If
+// the input packet contains too few samples to trigger a new output frame, no
+// output packet is generated (since zero-length packets are not legal since
+// they would result in timestamps that were equal, not strictly increasing).
+//
+// Output packet Timestamps are set to the beginning of each frame. This is to
+// allow calculators downstream from SpectrogramCalculator to have aligned
+// Timestamps regardless of a packet's signal length.
+//
+// Both frame_duration_seconds and frame_overlap_seconds will be
+// rounded to the nearest integer number of samples.  Conseqently, all output
+// frames will be based on the same number of input samples, and each
+// analysis frame will advance from its predecessor by the same time step.
+class SpectrogramCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input stream with TimeSeriesHeader.
+    );
+
+    SpectrogramCalculatorOptions spectrogram_options =
+        cc->Options<SpectrogramCalculatorOptions>();
+    if (!spectrogram_options.allow_multichannel_input()) {
+      if (spectrogram_options.output_type() ==
+          SpectrogramCalculatorOptions::COMPLEX) {
+        cc->Outputs().Index(0).Set<Eigen::MatrixXcf>(
+            // Complex spectrogram frames with TimeSeriesHeader.
+        );
+      } else {
+        cc->Outputs().Index(0).Set<Matrix>(
+            // Spectrogram frames with TimeSeriesHeader.
+        );
+      }
+    } else {
+      if (spectrogram_options.output_type() ==
+          SpectrogramCalculatorOptions::COMPLEX) {
+        cc->Outputs().Index(0).Set<std::vector<Eigen::MatrixXcf>>(
+            // Complex spectrogram frames with MultiStreamTimeSeriesHeader.
+        );
+      } else {
+        cc->Outputs().Index(0).Set<std::vector<Matrix>>(
+            // Spectrogram frames with MultiStreamTimeSeriesHeader.
+        );
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  // Returns FAIL if the input stream header is invalid.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Outputs at most one packet consisting of a single Matrix with one or
+  // more columns containing the spectral values from as many input frames
+  // as are completed by the input samples.  Always returns OK.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+  // Performs zero-padding and processing of any remaining samples
+  // if pad_final_packet is set.
+  // Returns OK.
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  Timestamp CurrentOutputTimestamp(CalculatorContext* cc) {
+    if (use_local_timestamp_) {
+      return cc->InputTimestamp();
+    }
+    return CumulativeOutputTimestamp();
+  }
+
+  Timestamp CumulativeOutputTimestamp() {
+    // Cumulative output timestamp is the *center* of the next frame to be
+    // emitted, hence delayed by half a window duration compared to relevant
+    // input timestamp.
+    return initial_input_timestamp_ +
+           round(cumulative_completed_frames_ * frame_step_samples() *
+                 Timestamp::kTimestampUnitsPerSecond / input_sample_rate_);
+  }
+
+  int frame_step_samples() const {
+    return frame_duration_samples_ - frame_overlap_samples_;
+  }
+
+  // Take the next set of input samples, already translated into a
+  // vector<float> and pass them to the spectrogram object.
+  // Convert the output of the spectrogram object into a Matrix (or an
+  // Eigen::MatrixXcf if complex-valued output is requested) and pass to
+  // MediaPipe output.
+  ::mediapipe::Status ProcessVector(const Matrix& input_stream,
+                                    CalculatorContext* cc);
+
+  // Templated function to process either real- or complex-output spectrogram.
+  template <class OutputMatrixType>
+  ::mediapipe::Status ProcessVectorToOutput(
+      const Matrix& input_stream,
+      const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
+      CalculatorContext* cc);
+
+  bool use_local_timestamp_;
+  double input_sample_rate_;
+  bool pad_final_packet_;
+  int frame_duration_samples_;
+  int frame_overlap_samples_;
+  // How many samples we've been passed, used for checking input time stamps.
+  int64 cumulative_input_samples_;
+  // How many frames we've emitted, used for calculating output time stamps.
+  int64 cumulative_completed_frames_;
+  Timestamp initial_input_timestamp_;
+  int num_input_channels_;
+  // How many frequency bins we emit (=N_FFT/2 + 1).
+  int num_output_channels_;
+  // Which output type?
+  int output_type_;
+  // Output type: mono or multichannel.
+  bool allow_multichannel_input_;
+  // Vector of Spectrogram objects, one for each channel.
+  std::vector<std::unique_ptr<audio_dsp::Spectrogram>> spectrogram_generators_;
+  // Fixed scale factor applied to output values (regardless of type).
+  double output_scale_;
+
+  static const float kLnPowerToDb;
+};
+REGISTER_CALCULATOR(SpectrogramCalculator);
+
+// Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0).
+const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
+
+::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
+  SpectrogramCalculatorOptions spectrogram_options =
+      cc->Options<SpectrogramCalculatorOptions>();
+
+  use_local_timestamp_ = spectrogram_options.use_local_timestamp();
+
+  if (spectrogram_options.frame_duration_seconds() <= 0.0) {
+    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+        << "Invalid or missing frame_duration_seconds.\n"
+           "frame_duration_seconds: "
+        << spectrogram_options.frame_overlap_seconds();
+  }
+  if (spectrogram_options.frame_overlap_seconds() >=
+      spectrogram_options.frame_duration_seconds()) {
+    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+        << "Invalid frame_overlap_seconds.\nframe_overlap_seconds: "
+        << spectrogram_options.frame_overlap_seconds()
+        << "\nframe_duration_seconds: "
+        << spectrogram_options.frame_duration_seconds();
+  }
+  if (spectrogram_options.frame_overlap_seconds() < 0.0) {
+    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+        << "Frame_overlap_seconds is < 0.0.\nframe_overlap_seconds: "
+        << spectrogram_options.frame_overlap_seconds();
+  }
+
+  TimeSeriesHeader input_header;
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+      cc->Inputs().Index(0).Header(), &input_header));
+
+  input_sample_rate_ = input_header.sample_rate();
+  num_input_channels_ = input_header.num_channels();
+
+  if (!spectrogram_options.allow_multichannel_input() &&
+      num_input_channels_ != 1) {
+    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+        << "The current setting only supports single-channel input. Please set "
+           "allow_multichannel_input.\n";
+  }
+
+  frame_duration_samples_ =
+      round(spectrogram_options.frame_duration_seconds() * input_sample_rate_);
+  frame_overlap_samples_ =
+      round(spectrogram_options.frame_overlap_seconds() * input_sample_rate_);
+
+  pad_final_packet_ = spectrogram_options.pad_final_packet();
+  output_type_ = spectrogram_options.output_type();
+  allow_multichannel_input_ = spectrogram_options.allow_multichannel_input();
+
+  output_scale_ = spectrogram_options.output_scale();
+
+  std::vector<double> window;
+  switch (spectrogram_options.window_type()) {
+    case SpectrogramCalculatorOptions::COSINE:
+      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                   &window);
+      break;
+    case SpectrogramCalculatorOptions::HANN:
+      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                 &window);
+      break;
+    case SpectrogramCalculatorOptions::HAMMING:
+      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                    &window);
+      break;
+  }
+
+  // Propagate settings down to the actual Spectrogram object.
+  spectrogram_generators_.clear();
+  for (int i = 0; i < num_input_channels_; i++) {
+    spectrogram_generators_.push_back(
+        std::unique_ptr<audio_dsp::Spectrogram>(new audio_dsp::Spectrogram()));
+    spectrogram_generators_[i]->Initialize(window, frame_step_samples());
+  }
+
+  num_output_channels_ =
+      spectrogram_generators_[0]->output_frequency_channels();
+  std::unique_ptr<TimeSeriesHeader> output_header(
+      new TimeSeriesHeader(input_header));
+  // Store the actual sample rate of the input audio in the TimeSeriesHeader
+  // so that subsequent calculators can figure out the frequency scale of
+  // our output.
+  output_header->set_audio_sample_rate(input_sample_rate_);
+  // Setup rest of output header.
+  output_header->set_num_channels(num_output_channels_);
+  output_header->set_sample_rate(input_sample_rate_ / frame_step_samples());
+  // Although we usually generate one output packet for each input
+  // packet, this might not be true for input packets whose size is smaller
+  // than the analysis window length.  So we clear output_header.packet_rate
+  // because we can't guarantee a constant packet rate.  Similarly, the number
+  // of output frames per packet depends on the input packet, so we also clear
+  // output_header.num_samples.
+  output_header->clear_packet_rate();
+  output_header->clear_num_samples();
+  if (!spectrogram_options.allow_multichannel_input()) {
+    cc->Outputs().Index(0).SetHeader(Adopt(output_header.release()));
+  } else {
+    std::unique_ptr<MultiStreamTimeSeriesHeader> multichannel_output_header(
+        new MultiStreamTimeSeriesHeader());
+    *multichannel_output_header->mutable_time_series_header() = *output_header;
+    multichannel_output_header->set_num_streams(num_input_channels_);
+    cc->Outputs().Index(0).SetHeader(
+        Adopt(multichannel_output_header.release()));
+  }
+  cumulative_completed_frames_ = 0;
+  initial_input_timestamp_ = Timestamp::Unstarted();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SpectrogramCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+  }
+
+  const Matrix& input_stream = cc->Inputs().Index(0).Get<Matrix>();
+  if (input_stream.rows() != num_input_channels_) {
+    ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+        << "Number of input channels do not correspond to the number of rows "
+        << "in the input matrix: " << num_input_channels_ << "channels vs "
+        << input_stream.rows() << " rows";
+  }
+
+  cumulative_input_samples_ += input_stream.cols();
+
+  return ProcessVector(input_stream, cc);
+}
+
+template <class OutputMatrixType>
+::mediapipe::Status SpectrogramCalculator::ProcessVectorToOutput(
+    const Matrix& input_stream,
+    const OutputMatrixType postprocess_output_fn(const OutputMatrixType&),
+    CalculatorContext* cc) {
+  std::unique_ptr<std::vector<OutputMatrixType>> spectrogram_matrices(
+      new std::vector<OutputMatrixType>());
+  std::vector<std::vector<typename OutputMatrixType::Scalar>> output_vectors;
+
+  // Compute a spectrogram for each channel.
+  int num_output_time_frames;
+  for (int channel = 0; channel < input_stream.rows(); ++channel) {
+    output_vectors.clear();
+
+    // Copy one row (channel) of the input matrix into the std::vector.
+    std::vector<float> input_vector(input_stream.cols());
+    Eigen::Map<Matrix>(&input_vector[0], 1, input_vector.size()) =
+        input_stream.row(channel);
+
+    if (!spectrogram_generators_[channel]->ComputeSpectrogram(
+            input_vector, &output_vectors)) {
+      return ::mediapipe::Status(mediapipe::StatusCode::kInternal,
+                                 "Spectrogram returned failure");
+    }
+    if (channel == 0) {
+      // Record the number of time frames we expect from each channel.
+      num_output_time_frames = output_vectors.size();
+    } else {
+      RET_CHECK_EQ(output_vectors.size(), num_output_time_frames)
+          << "Inconsistent spectrogram time frames for channel " << channel;
+    }
+    // Skip remaining processing if there are too few input samples to trigger
+    // any output frames.
+    if (!output_vectors.empty()) {
+      // Translate the returned values into a matrix of output frames.
+      OutputMatrixType output_frames(num_output_channels_,
+                                     output_vectors.size());
+      for (int frame = 0; frame < output_vectors.size(); ++frame) {
+        Eigen::Map<const OutputMatrixType> frame_map(
+            &output_vectors[frame][0], output_vectors[frame].size(), 1);
+        // The underlying dsp object returns squared magnitudes; here
+        // we optionally translate to linear magnitude or dB.
+        output_frames.col(frame) =
+            output_scale_ * postprocess_output_fn(frame_map);
+      }
+      spectrogram_matrices->push_back(output_frames);
+    }
+  }
+  // If the input is very short, there may not be enough accumulated,
+  // unprocessed samples to cause any new frames to be generated by
+  // the spectrogram object.  If so, we don't want to emit
+  // a packet at all.
+  if (!spectrogram_matrices->empty()) {
+    RET_CHECK_EQ(spectrogram_matrices->size(), input_stream.rows())
+        << "Inconsistent number of spectrogram channels.";
+    if (allow_multichannel_input_) {
+      cc->Outputs().Index(0).Add(spectrogram_matrices.release(),
+                                 CurrentOutputTimestamp(cc));
+    } else {
+      cc->Outputs().Index(0).Add(
+          new OutputMatrixType(spectrogram_matrices->at(0)),
+          CurrentOutputTimestamp(cc));
+    }
+    cumulative_completed_frames_ += output_vectors.size();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SpectrogramCalculator::ProcessVector(
+    const Matrix& input_stream, CalculatorContext* cc) {
+  switch (output_type_) {
+    // These blocks deliberately ignore clang-format to preserve the
+    // "silhouette" of the different cases.
+    // clang-format off
+    case SpectrogramCalculatorOptions::COMPLEX: {
+      return ProcessVectorToOutput(
+          input_stream,
+          +[](const Eigen::MatrixXcf& col) -> const Eigen::MatrixXcf {
+            return col;
+          }, cc);
+    }
+    case SpectrogramCalculatorOptions::SQUARED_MAGNITUDE: {
+      return ProcessVectorToOutput(
+          input_stream,
+          +[](const Matrix& col) -> const Matrix {
+            return col;
+          }, cc);
+    }
+    case SpectrogramCalculatorOptions::LINEAR_MAGNITUDE: {
+      return ProcessVectorToOutput(
+          input_stream,
+          +[](const Matrix& col) -> const Matrix {
+            return col.array().sqrt().matrix();
+          }, cc);
+    }
+    case SpectrogramCalculatorOptions::DECIBELS: {
+      return ProcessVectorToOutput(
+          input_stream,
+          +[](const Matrix& col) -> const Matrix {
+            return kLnPowerToDb * col.array().log().matrix();
+          }, cc);
+    }
+    // clang-format on
+    default: {
+      return ::mediapipe::Status(mediapipe::StatusCode::kInvalidArgument,
+                                 "Unrecognized spectrogram output type.");
+    }
+  }
+}
+
+::mediapipe::Status SpectrogramCalculator::Close(CalculatorContext* cc) {
+  if (cumulative_input_samples_ > 0 && pad_final_packet_) {
+    // We can flush any remaining samples by sending frame_step_samples - 1
+    // zeros to the Process method, and letting it do its thing,
+    // UNLESS we have fewer than one window's worth of samples, in which case
+    // we pad to exactly one frame_duration_samples.
+    // Release the memory for the Spectrogram objects.
+    int required_padding_samples = frame_step_samples() - 1;
+    if (cumulative_input_samples_ < frame_duration_samples_) {
+      required_padding_samples =
+          frame_duration_samples_ - cumulative_input_samples_;
+    }
+    return ProcessVector(
+        Matrix::Zero(num_input_channels_, required_padding_samples), cc);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -0,0 +1,76 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message SpectrogramCalculatorOptions {
+  extend CalculatorOptions {
+    optional SpectrogramCalculatorOptions ext = 76186688;
+  }
+
+  // Options mirror those of TimeSeriesFramerCalculator.
+
+  // Analysis window duration in seconds.  Required.  Must be greater than 0.
+  // (Note: the spectrogram DFT length will be the smallest power-of-2
+  // sample count that can hold this duration.)
+  optional double frame_duration_seconds = 1;
+
+  // Duration of overlap between adjacent windows.
+  // Hence, frame_rate = 1/(frame_duration_seconds - frame_overlap_seconds).
+  // Required that 0 <= frame_overlap_seconds <  frame_duration_seconds.
+  optional double frame_overlap_seconds = 2 [default = 0.0];
+
+  // Whether to pad the final packet with zeros.  If true, guarantees that
+  // all input samples will output.  If set to false, any partial packet
+  // at the end of the stream will be dropped.
+  optional bool pad_final_packet = 3 [default = true];
+
+  // Output value type can be squared-magnitude, linear-magnitude,
+  // deciBels (dB, = 20*log10(linear_magnitude)), or std::complex.
+  enum OutputType {
+    SQUARED_MAGNITUDE = 0;
+    LINEAR_MAGNITUDE = 1;
+    DECIBELS = 2;
+    COMPLEX = 3;
+  }
+  optional OutputType output_type = 4 [default = SQUARED_MAGNITUDE];
+
+  // If set to true then the output will be a vector of spectrograms, one for
+  // each channel and the stream will have a MultiStreamTimeSeriesHeader.
+  optional bool allow_multichannel_input = 5 [default = false];
+
+  // Which window to use when computing the FFT.
+  enum WindowType {
+    HANN = 0;
+    HAMMING = 1;
+    COSINE = 2;
+  }
+  optional WindowType window_type = 6 [default = HANN];
+
+  // Support a fixed multiplicative scaling of the output.  This is applied
+  // uniformly regardless of output type (i.e., even dBs are multiplied, not
+  // offset).
+  optional double output_scale = 7 [default = 1.0];
+
+  // If use_local_timestamp is true, the output packet's timestamp is based on
+  // the last sample of the packet and it's inferred from the latest input
+  // packet's timestamp.  If false, the output packet's timestamp is based on
+  // the cumulative timestamping, which is inferred from the intial input
+  // timestamp and the cumulative number of samples.
+  optional bool use_local_timestamp = 8 [default = false];
+}

mediapipe/calculators/audio/spectrogram_calculator_test.cc

@@ -0,0 +1,895 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <cmath>
+#include <complex>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "Eigen/Core"
+#include "audio/dsp/number_util.h"
+#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/benchmark.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+namespace {
+
+const int kInitialTimestampOffsetMicroseconds = 4;
+
+class SpectrogramCalculatorTest
+    : public TimeSeriesCalculatorTest<SpectrogramCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "SpectrogramCalculator";
+    input_sample_rate_ = 4000.0;
+    num_input_channels_ = 1;
+  }
+
+  // Initializes and runs the test graph.
+  ::mediapipe::Status Run() {
+    // Now that options are set, we can set up some internal constants.
+    frame_duration_samples_ =
+        round(options_.frame_duration_seconds() * input_sample_rate_);
+    frame_step_samples_ =
+        frame_duration_samples_ -
+        round(options_.frame_overlap_seconds() * input_sample_rate_);
+    // The magnitude of the 0th FFT bin (DC) should be sum(input.*window);
+    // for an input identically 1.0, this is just sum(window).  The average
+    // value of our Hann window is 0.5, hence this is the expected squared-
+    // magnitude output value in the DC bin for constant input of 1.0.
+    expected_dc_squared_magnitude_ =
+        pow((static_cast<float>(frame_duration_samples_) * 0.5), 2.0);
+
+    return RunGraph();
+  }
+
+  // Creates test multichannel input with specified packet sizes and containing
+  // a constant-frequency sinusoid that maintains phase between adjacent
+  // packets.
+  void SetupCosineInputPackets(const std::vector<int>& packet_sizes_samples,
+                               float cosine_frequency_hz) {
+    int total_num_input_samples = 0;
+    for (int packet_size_samples : packet_sizes_samples) {
+      double packet_start_time_seconds =
+          kInitialTimestampOffsetMicroseconds * 1e-6 +
+          total_num_input_samples / input_sample_rate_;
+      double packet_end_time_seconds =
+          packet_start_time_seconds + packet_size_samples / input_sample_rate_;
+      double angular_freq = 2 * M_PI * cosine_frequency_hz;
+      Matrix* packet_data =
+          new Matrix(num_input_channels_, packet_size_samples);
+      // Use Eigen's vectorized cos() function to fill the vector with a
+      // sinusoid of appropriate frequency & phase.
+      for (int i = 0; i < num_input_channels_; i++) {
+        packet_data->row(i) =
+            Eigen::ArrayXf::LinSpaced(packet_size_samples,
+                                      packet_start_time_seconds * angular_freq,
+                                      packet_end_time_seconds * angular_freq)
+                .cos()
+                .transpose();
+      }
+      int64 input_timestamp = round(packet_start_time_seconds *
+                                    Timestamp::kTimestampUnitsPerSecond);
+      AppendInputPacket(packet_data, input_timestamp);
+      total_num_input_samples += packet_size_samples;
+    }
+  }
+
+  // Setup a sequence of input packets of specified sizes, each filled
+  // with samples of 1.0.
+  void SetupConstantInputPackets(const std::vector<int>& packet_sizes_samples) {
+    // A 0 Hz cosine is identically 1.0 for all samples.
+    SetupCosineInputPackets(packet_sizes_samples, 0.0);
+  }
+
+  // Setup a sequence of input packets of specified sizes, each containing a
+  // single sample of 1.0 at a specified offset.
+  void SetupImpulseInputPackets(
+      const std::vector<int>& packet_sizes_samples,
+      const std::vector<int>& impulse_offsets_samples) {
+    int total_num_input_samples = 0;
+    for (int i = 0; i < packet_sizes_samples.size(); ++i) {
+      double packet_start_time_seconds =
+          kInitialTimestampOffsetMicroseconds * 1e-6 +
+          total_num_input_samples / input_sample_rate_;
+      int64 input_timestamp = round(packet_start_time_seconds *
+                                    Timestamp::kTimestampUnitsPerSecond);
+      std::unique_ptr<Matrix> impulse(
+          new Matrix(Matrix::Zero(1, packet_sizes_samples[i])));
+      (*impulse)(0, impulse_offsets_samples[i]) = 1.0;
+      AppendInputPacket(impulse.release(), input_timestamp);
+      total_num_input_samples += packet_sizes_samples[i];
+    }
+  }
+
+  // Creates test multichannel input with specified packet sizes and containing
+  // constant input packets for the even channels and constant-frequency
+  // sinusoid that maintains phase between adjacent packets for the odd
+  // channels.
+  void SetupMultichannelInputPackets(
+      const std::vector<int>& packet_sizes_samples, float cosine_frequency_hz) {
+    int total_num_input_samples = 0;
+    for (int packet_size_samples : packet_sizes_samples) {
+      double packet_start_time_seconds =
+          kInitialTimestampOffsetMicroseconds * 1e-6 +
+          total_num_input_samples / input_sample_rate_;
+      double packet_end_time_seconds =
+          packet_start_time_seconds + packet_size_samples / input_sample_rate_;
+      double angular_freq;
+      Matrix* packet_data =
+          new Matrix(num_input_channels_, packet_size_samples);
+      // Use Eigen's vectorized cos() function to fill the vector with a
+      // sinusoid of appropriate frequency & phase.
+      for (int i = 0; i < num_input_channels_; i++) {
+        if (i % 2 == 0) {
+          angular_freq = 0;
+        } else {
+          angular_freq = 2 * M_PI * cosine_frequency_hz;
+        }
+        packet_data->row(i) =
+            Eigen::ArrayXf::LinSpaced(packet_size_samples,
+                                      packet_start_time_seconds * angular_freq,
+                                      packet_end_time_seconds * angular_freq)
+                .cos()
+                .transpose();
+      }
+      int64 input_timestamp = round(packet_start_time_seconds *
+                                    Timestamp::kTimestampUnitsPerSecond);
+      AppendInputPacket(packet_data, input_timestamp);
+      total_num_input_samples += packet_size_samples;
+    }
+  }
+
+  // Return vector of the numbers of frames in each output packet.
+  std::vector<int> OutputFramesPerPacket() {
+    std::vector<int> frame_counts;
+    for (const Packet& packet : output().packets) {
+      const Matrix& matrix = packet.Get<Matrix>();
+      frame_counts.push_back(matrix.cols());
+    }
+    return frame_counts;
+  }
+
+  // Checks output headers and Timestamps.
+  void CheckOutputHeadersAndTimestamps() {
+    const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
+
+    TimeSeriesHeader expected_header = input().header.Get<TimeSeriesHeader>();
+    expected_header.set_num_channels(fft_size / 2 + 1);
+    // The output header sample rate should depend on the output frame step.
+    expected_header.set_sample_rate(input_sample_rate_ / frame_step_samples_);
+    // SpectrogramCalculator stores the sample rate of the input in
+    // the TimeSeriesHeader.
+    expected_header.set_audio_sample_rate(input_sample_rate_);
+    // We expect the output header to have num_samples and packet_rate unset.
+    expected_header.clear_num_samples();
+    expected_header.clear_packet_rate();
+    if (!options_.allow_multichannel_input()) {
+      ExpectOutputHeaderEquals(expected_header);
+    } else {
+      EXPECT_THAT(output()
+                      .header.template Get<MultiStreamTimeSeriesHeader>()
+                      .time_series_header(),
+                  mediapipe::EqualsProto(expected_header));
+      EXPECT_THAT(output()
+                      .header.template Get<MultiStreamTimeSeriesHeader>()
+                      .num_streams(),
+                  num_input_channels_);
+    }
+
+    int cumulative_output_frames = 0;
+    // The timestamps coming out of the spectrogram correspond to the
+    // middle of the first frame's window, hence frame_duration_samples_/2
+    // term.  We use frame_duration_samples_ because that is how it is
+    // actually quantized inside spectrogram.
+    const double packet_timestamp_offset_seconds =
+        kInitialTimestampOffsetMicroseconds * 1e-6;
+    const double frame_step_seconds = frame_step_samples_ / input_sample_rate_;
+
+    Timestamp initial_timestamp = Timestamp::Unstarted();
+
+    for (const Packet& packet : output().packets) {
+      // This is the timestamp we expect based on how the spectrogram should
+      // behave (advancing by one step's worth of input samples each frame).
+      const double expected_timestamp_seconds =
+          packet_timestamp_offset_seconds +
+          cumulative_output_frames * frame_step_seconds;
+      const int64 expected_timestamp_ticks =
+          expected_timestamp_seconds * Timestamp::kTimestampUnitsPerSecond;
+      EXPECT_EQ(expected_timestamp_ticks, packet.Timestamp().Value());
+      // Accept the timestamp of the first packet as the baseline for checking
+      // the remainder.
+      if (initial_timestamp == Timestamp::Unstarted()) {
+        initial_timestamp = packet.Timestamp();
+      }
+      // Also check that the timestamp is consistent with the sample_rate
+      // in the output stream's TimeSeriesHeader.
+      EXPECT_TRUE(time_series_util::LogWarningIfTimestampIsInconsistent(
+          packet.Timestamp(), initial_timestamp, cumulative_output_frames,
+          expected_header.sample_rate()));
+      if (!options_.allow_multichannel_input()) {
+        if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) {
+          const Eigen::MatrixXcf& matrix = packet.Get<Eigen::MatrixXcf>();
+          cumulative_output_frames += matrix.cols();
+        } else {
+          const Matrix& matrix = packet.Get<Matrix>();
+          cumulative_output_frames += matrix.cols();
+        }
+      } else {
+        if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) {
+          const Eigen::MatrixXcf& matrix =
+              packet.Get<std::vector<Eigen::MatrixXcf>>().at(0);
+          cumulative_output_frames += matrix.cols();
+        } else {
+          const Matrix& matrix = packet.Get<std::vector<Matrix>>().at(0);
+          cumulative_output_frames += matrix.cols();
+        }
+      }
+    }
+  }
+
+  // Verify that the bin corresponding to the specified frequency
+  // is the largest one in one particular frame of a single packet.
+  void CheckPeakFrequencyInPacketFrame(const Packet& packet, int frame,
+                                       float frequency) {
+    const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
+    const int target_bin =
+        round((frequency / input_sample_rate_) * static_cast<float>(fft_size));
+
+    const Matrix& matrix = packet.Get<Matrix>();
+    // Stop here if the requested frame is not in this packet.
+    ASSERT_GT(matrix.cols(), frame);
+
+    int actual_largest_bin;
+    matrix.col(frame).maxCoeff(&actual_largest_bin);
+    EXPECT_EQ(actual_largest_bin, target_bin);
+  }
+
+  // Verify that the bin corresponding to the specified frequency
+  // is the largest one in one particular frame of a single spectrogram Matrix.
+  void CheckPeakFrequencyInMatrix(const Matrix& matrix, int frame,
+                                  float frequency) {
+    const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_);
+    const int target_bin =
+        round((frequency / input_sample_rate_) * static_cast<float>(fft_size));
+
+    // Stop here if the requested frame is not in this packet.
+    ASSERT_GT(matrix.cols(), frame);
+
+    int actual_largest_bin;
+    matrix.col(frame).maxCoeff(&actual_largest_bin);
+    EXPECT_EQ(actual_largest_bin, target_bin);
+  }
+
+  int frame_duration_samples_;
+  int frame_step_samples_;
+  // Expected DC output for a window of pure 1.0, set when window length
+  // is set.
+  float expected_dc_squared_magnitude_;
+};
+
+TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationNoOverlap) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {500, 200};
+  const std::vector<int> expected_output_packet_sizes = {5, 2};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationSomeOverlap) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {500, 200};
+  // complete_output_frames = 1 + floor((input_samples - window_length)/step)
+  //   = 1 + floor((500 - 100)/40) = 1 + 10 = 11 for the first packet
+  //   = 1 + floor((700 - 100)/40) = 1 + 15 = 16 for the whole stream
+  // so expect 16 - 11 = 5 in the second packet.
+  const std::vector<int> expected_output_packet_sizes = {11, 5};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, NonintegerFrameDurationAndOverlap) {
+  options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(58.4 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {500, 200};
+  // now frame_duration_samples will be 99 (rounded), and frame_step_samples
+  // will be (99-58) = 41, so the first packet of 500 samples will generate
+  // 1 + floor(500-99)/41 = 10 samples.
+  const std::vector<int> expected_output_packet_sizes = {10, 5};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, ShortInitialPacketNoOverlap) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {90, 100, 110};
+  // The first input packet is too small to generate any frames,
+  // but zero-length packets would result in a timestamp monotonicity
+  // violation, so they are suppressed.  Thus, only the second and third
+  // input packets generate output packets.
+  const std::vector<int> expected_output_packet_sizes = {1, 2};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, TrailingSamplesNoPad) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {140, 90};
+  const std::vector<int> expected_output_packet_sizes = {2, 2};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, NoTrailingSamplesWithPad) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(true);
+  const std::vector<int> input_packet_sizes = {140, 80};
+  const std::vector<int> expected_output_packet_sizes = {2, 2};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, TrailingSamplesWithPad) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(true);
+  const std::vector<int> input_packet_sizes = {140, 90};
+  // In contrast to NoTrailingSamplesWithPad and TrailingSamplesNoPad,
+  // this time we get an extra frame in an extra final packet.
+  const std::vector<int> expected_output_packet_sizes = {2, 2, 1};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, VeryShortInputWillPad) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(true);
+  const std::vector<int> input_packet_sizes = {30};
+  const std::vector<int> expected_output_packet_sizes = {1};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, VeryShortInputZeroOutputFramesIfNoPad) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  const std::vector<int> input_packet_sizes = {90};
+  const std::vector<int> expected_output_packet_sizes = {};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, DCSignalIsPeakBin) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  const std::vector<int> input_packet_sizes = {140};  // Gives 2 output frames.
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  const float dc_frequency_hz = 0.0;
+  CheckPeakFrequencyInPacketFrame(output().packets[0], 0, dc_frequency_hz);
+  CheckPeakFrequencyInPacketFrame(output().packets[0], 1, dc_frequency_hz);
+}
+
+TEST_F(SpectrogramCalculatorTest, A440ToneIsPeakBin) {
+  const std::vector<int> input_packet_sizes = {
+      460};  // 100 + 9*40 for 10 frames.
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  int num_output_frames = output().packets[0].Get<Matrix>().cols();
+  for (int frame = 0; frame < num_output_frames; ++frame) {
+    CheckPeakFrequencyInPacketFrame(output().packets[0], frame,
+                                    tone_frequency_hz);
+  }
+}
+
+TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRight) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
+                  expected_dc_squared_magnitude_);
+}
+
+TEST_F(SpectrogramCalculatorTest, DefaultOutputIsSquaredMagnitude) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  // Let the output_type be its default
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
+                  expected_dc_squared_magnitude_);
+}
+
+TEST_F(SpectrogramCalculatorTest, LinearMagnitudeOutputLooksRight) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::LINEAR_MAGNITUDE);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
+                  std::sqrt(expected_dc_squared_magnitude_));
+}
+
+TEST_F(SpectrogramCalculatorTest, DbMagnitudeOutputLooksRight) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
+                  10.0 * std::log10(expected_dc_squared_magnitude_));
+}
+
+TEST_F(SpectrogramCalculatorTest, OutputScalingLooksRight) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS);
+  double output_scale = 2.5;
+  options_.set_output_scale(output_scale);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(
+      output().packets[0].Get<Matrix>()(0, 0),
+      output_scale * 10.0 * std::log10(expected_dc_squared_magnitude_));
+}
+
+TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRight) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Setup packets with DC input (non-zero constant value).
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_FLOAT_EQ(std::norm(output().packets[0].Get<Eigen::MatrixXcf>()(0, 0)),
+                  expected_dc_squared_magnitude_);
+}
+
+TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRightForImpulses) {
+  const int frame_size_samples = 100;
+  options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_);
+  options_.set_frame_overlap_seconds(0.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
+  const std::vector<int> input_packet_sizes = {frame_size_samples,
+                                               frame_size_samples};
+  const std::vector<int> input_packet_impulse_offsets = {49, 50};
+
+  InitializeGraph();
+  FillInputHeader();
+
+  // Make two impulse packets offset one sample from each other
+  SetupImpulseInputPackets(input_packet_sizes, input_packet_impulse_offsets);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  const int num_buckets =
+      (audio_dsp::NextPowerOfTwo(frame_size_samples) / 2) + 1;
+  const float precision = 0.01f;
+  auto norm_fn = [](const std::complex<float>& cf) { return std::norm(cf); };
+
+  // Both impulses should have (approximately) constant power across all
+  // frequency bins
+  EXPECT_TRUE(output()
+                  .packets[0]
+                  .Get<Eigen::MatrixXcf>()
+                  .unaryExpr(norm_fn)
+                  .isApproxToConstant(1.0f, precision));
+  EXPECT_TRUE(output()
+                  .packets[1]
+                  .Get<Eigen::MatrixXcf>()
+                  .unaryExpr(norm_fn)
+                  .isApproxToConstant(1.0f, precision));
+
+  // Because the second Packet's impulse is delayed by exactly one sample with
+  // respect to the first Packet's impulse, the second impulse should have
+  // greater phase, and in the highest frequency bin, the real part should
+  // (approximately) flip sign from the first Packet to the second
+  EXPECT_LT(std::arg(output().packets[0].Get<Eigen::MatrixXcf>()(1, 0)),
+            std::arg(output().packets[1].Get<Eigen::MatrixXcf>()(1, 0)));
+  const float highest_bucket_real_ratio =
+      output().packets[0].Get<Eigen::MatrixXcf>()(num_buckets - 1, 0).real() /
+      output().packets[1].Get<Eigen::MatrixXcf>()(num_buckets - 1, 0).real();
+  EXPECT_NEAR(highest_bucket_real_ratio, -1.0f, precision);
+}
+
+TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRightForNonDC) {
+  const int frame_size_samples = 100;
+  options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE);
+  const std::vector<int> input_packet_sizes = {140};
+
+  InitializeGraph();
+  FillInputHeader();
+  // Make the tone have an integral number of cycles within the window
+  const int target_bin = 16;
+  const int fft_size = audio_dsp::NextPowerOfTwo(frame_size_samples);
+  const float tone_frequency_hz = target_bin * (input_sample_rate_ / fft_size);
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  // For a non-DC bin, the magnitude will be split between positive and
+  // negative frequency bins, so it should about be half-magnitude
+  // = quarter-power.
+  // It's not quite exact because of the interference from the hann(100)
+  // spread from the negative-frequency half.
+  EXPECT_GT(output().packets[0].Get<Matrix>()(target_bin, 0),
+            0.98 * expected_dc_squared_magnitude_ / 4.0);
+  EXPECT_LT(output().packets[0].Get<Matrix>()(target_bin, 0),
+            1.02 * expected_dc_squared_magnitude_ / 4.0);
+}
+
+TEST_F(SpectrogramCalculatorTest, ZeroOutputsForZeroInputsWithPaddingEnabled) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(true);
+  const std::vector<int> input_packet_sizes = {};
+  const std::vector<int> expected_output_packet_sizes = {};
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
+}
+
+TEST_F(SpectrogramCalculatorTest, NumChannelsIsRight) {
+  const std::vector<int> input_packet_sizes = {460};
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  options_.set_allow_multichannel_input(true);
+  num_input_channels_ = 3;
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  EXPECT_EQ(output().packets[0].Get<std::vector<Matrix>>().size(),
+            num_input_channels_);
+}
+
+TEST_F(SpectrogramCalculatorTest, NumSamplesAndPacketRateAreCleared) {
+  num_input_samples_ = 500;
+  input_packet_rate_ = 1.0;
+  const std::vector<int> input_packet_sizes = {num_input_samples_};
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(0.0);
+  options_.set_pad_final_packet(false);
+
+  InitializeGraph();
+  FillInputHeader();
+  SetupConstantInputPackets(input_packet_sizes);
+
+  MP_ASSERT_OK(Run());
+
+  const TimeSeriesHeader& output_header =
+      output().header.Get<TimeSeriesHeader>();
+  EXPECT_FALSE(output_header.has_num_samples());
+  EXPECT_FALSE(output_header.has_packet_rate());
+}
+
+TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramSizesAreRight) {
+  const std::vector<int> input_packet_sizes = {420};  // less than 10 frames
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  options_.set_allow_multichannel_input(true);
+  num_input_channels_ = 10;
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
+  EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
+  int spectrogram_num_rows = spectrograms[0].rows();
+  int spectrogram_num_cols = spectrograms[0].cols();
+  for (int i = 1; i < num_input_channels_; i++) {
+    EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
+    EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
+  }
+}
+
+TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramValuesAreRight) {
+  const std::vector<int> input_packet_sizes = {
+      460};  // 100 + 9*40 for 10 frames.
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_allow_multichannel_input(true);
+  num_input_channels_ = 10;
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupMultichannelInputPackets(input_packet_sizes, tone_frequency_hz);
+
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
+  int num_output_frames = spectrograms[0].cols();
+  for (int i = 0; i < num_input_channels_; i++) {
+    for (int frame = 0; frame < num_output_frames; ++frame) {
+      if (i % 2 == 0) {
+        CheckPeakFrequencyInMatrix(spectrograms[i], frame, 0);
+      } else {
+        CheckPeakFrequencyInMatrix(spectrograms[i], frame, tone_frequency_hz);
+      }
+    }
+  }
+}
+
+TEST_F(SpectrogramCalculatorTest, MultichannelHandlesShortInitialPacket) {
+  // First packet is less than one frame, but second packet should trigger a
+  // complete frame from all channels.
+  const std::vector<int> input_packet_sizes = {50, 50};
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  options_.set_allow_multichannel_input(true);
+  num_input_channels_ = 2;
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
+  EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
+  int spectrogram_num_rows = spectrograms[0].rows();
+  int spectrogram_num_cols = spectrograms[0].cols();
+  for (int i = 1; i < num_input_channels_; i++) {
+    EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
+    EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
+  }
+}
+
+TEST_F(SpectrogramCalculatorTest,
+       MultichannelComplexHandlesShortInitialPacket) {
+  // First packet is less than one frame, but second packet should trigger a
+  // complete frame from all channels, even for complex output.
+  const std::vector<int> input_packet_sizes = {50, 50};
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(60.0 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+  options_.set_allow_multichannel_input(true);
+  options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX);
+  num_input_channels_ = 2;
+  InitializeGraph();
+  FillInputHeader();
+  const float tone_frequency_hz = 440.0;
+  SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
+  MP_ASSERT_OK(Run());
+
+  CheckOutputHeadersAndTimestamps();
+  auto spectrograms = output().packets[0].Get<std::vector<Eigen::MatrixXcf>>();
+  EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_);
+  int spectrogram_num_rows = spectrograms[0].rows();
+  int spectrogram_num_cols = spectrograms[0].cols();
+  for (int i = 1; i < num_input_channels_; i++) {
+    EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows());
+    EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols());
+  }
+}
+
+void BM_ProcessDC(benchmark::State& state) {
+  CalculatorGraphConfig::Node node_config;
+  node_config.set_calculator("SpectrogramCalculator");
+  node_config.add_input_stream("input_audio");
+  node_config.add_output_stream("output_spectrogram");
+
+  SpectrogramCalculatorOptions* options =
+      node_config.mutable_options()->MutableExtension(
+          SpectrogramCalculatorOptions::ext);
+  options->set_frame_duration_seconds(0.010);
+  options->set_frame_overlap_seconds(0.0);
+  options->set_pad_final_packet(false);
+  *node_config.mutable_options()->MutableExtension(
+      SpectrogramCalculatorOptions::ext) = *options;
+
+  int num_input_channels = 1;
+  int packet_size_samples = 1600000;
+  TimeSeriesHeader* header = new TimeSeriesHeader();
+  header->set_sample_rate(16000.0);
+  header->set_num_channels(num_input_channels);
+
+  CalculatorRunner runner(node_config);
+  runner.MutableInputs()->Index(0).header = Adopt(header);
+
+  Matrix* payload = new Matrix(
+      Matrix::Constant(num_input_channels, packet_size_samples, 1.0));
+  Timestamp timestamp = Timestamp(0);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(payload).At(timestamp));
+
+  for (auto _ : state) {
+    ASSERT_TRUE(runner.Run().ok());
+  }
+
+  const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
+  const Matrix& output_matrix = output.packets[0].Get<Matrix>();
+  LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
+            << output_matrix.cols();
+  LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
+            << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
+            << output_matrix(3, 0);
+}
+
+BENCHMARK(BM_ProcessDC);
+
+}  // anonymous namespace
+}  // namespace mediapipe

mediapipe/calculators/audio/stabilized_log_calculator.cc

@@ -0,0 +1,99 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines StabilizedLogCalculator.
+
+#include <cmath>
+#include <memory>
+#include <string>
+
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "StabilizedLogCalculator"
+//   input_stream: "input_time_series"
+//   output_stream: "stabilized_log_time_series"
+//   options {
+//     [mediapipe.StabilizedLogCalculatorOptions.ext] {
+//       stabilizer: .00001
+//       check_nonnegativity: true
+//     }
+//   }
+// }
+class StabilizedLogCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input stream with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Output stabilized log stream with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    StabilizedLogCalculatorOptions stabilized_log_calculator_options =
+        cc->Options<StabilizedLogCalculatorOptions>();
+
+    stabilizer_ = stabilized_log_calculator_options.stabilizer();
+    output_scale_ = stabilized_log_calculator_options.output_scale();
+    check_nonnegativity_ =
+        stabilized_log_calculator_options.check_nonnegativity();
+    CHECK_GE(stabilizer_, 0.0)
+        << "stabilizer must be >= 0.0, received a value of " << stabilizer_;
+
+    // If the input packets have a header, propagate the header to the output.
+    if (!cc->Inputs().Index(0).Header().IsEmpty()) {
+      TimeSeriesHeader input_header;
+      MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+          cc->Inputs().Index(0).Header(), &input_header));
+      cc->Outputs().Index(0).SetHeader(
+          Adopt(new TimeSeriesHeader(input_header)));
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
+    if (input_matrix.array().isNaN().any()) {
+      return ::mediapipe::InvalidArgumentError("NaN input to log operation.");
+    }
+    if (check_nonnegativity_) {
+      if (input_matrix.minCoeff() < 0.0) {
+        return ::mediapipe::OutOfRangeError("Negative input to log operation.");
+      }
+    }
+    std::unique_ptr<Matrix> output_frame(new Matrix(
+        output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
+    cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float stabilizer_;
+  bool check_nonnegativity_;
+  double output_scale_;
+};
+REGISTER_CALCULATOR(StabilizedLogCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/stabilized_log_calculator.proto

@@ -0,0 +1,37 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message StabilizedLogCalculatorOptions {
+  extend CalculatorOptions {
+    optional StabilizedLogCalculatorOptions ext = 101978339;
+  }
+
+  // The calculator computes log(x + stabilizer). stabilizer must be >=
+  // 0, with 0 indicating a lack of stabilization.
+  optional float stabilizer = 1 [default = .00001];
+
+  // If true, CHECK that all input values in are >= 0. If false, the
+  // code will take the log of the potentially negative input values
+  // plus the stabilizer.
+  optional bool check_nonnegativity = 2 [default = true];
+
+  // Support a fixed multiplicative scaling of the output.
+  optional double output_scale = 3 [default = 1.0];
+}

mediapipe/calculators/audio/stabilized_log_calculator_test.cc

@@ -0,0 +1,141 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <cmath>
+
+#include "Eigen/Core"
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+const float kStabilizer = 0.1;
+const int kNumChannels = 3;
+const int kNumSamples = 10;
+
+class StabilizedLogCalculatorTest
+    : public TimeSeriesCalculatorTest<StabilizedLogCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "StabilizedLogCalculator";
+    options_.set_stabilizer(kStabilizer);
+
+    input_sample_rate_ = 8000.0;
+    num_input_channels_ = kNumChannels;
+    num_input_samples_ = kNumSamples;
+  }
+
+  void RunGraphNoReturn() { MP_ASSERT_OK(RunGraph()); }
+};
+
+TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
+  const int kNumPackets = 5;
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        (input_data_matrices[output_packet].array() + kStabilizer).log(),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
+  const int kNumPackets = 5;
+  double output_scale = 2.5;
+  options_.set_output_scale(output_scale);
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        output_scale *
+            ((input_data_matrices[output_packet].array() + kStabilizer).log()),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)),
+                    0 /* timestamp */);
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  ExpectApproximatelyEqual(
+      Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(),
+      runner_->Outputs().Index(0).packets[0].Get<Matrix>());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NanValuesReturnError) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, std::nanf(""))),
+      0 /* timestamp */);
+  ASSERT_FALSE(RunGraph().ok());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesReturnError) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  ASSERT_FALSE(RunGraph().ok());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {
+  options_.set_check_nonnegativity(false);
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  MP_ASSERT_OK(RunGraph());
+  // Results are undefined.
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/testdata/BUILD

@@ -16,11 +16,12 @@
 licenses(["notice"])  # Apache 2.0
 
 filegroup(
-    name = "test_videos",
+    name = "test_audios",
     srcs = [
-        "format_FLV_H264_AAC.video",
+        "sine_wave_1k_44100_mono_2_sec_wav.audio",
-        "format_MKV_VP8_VORBIS.video",
+        "sine_wave_1k_44100_stereo_2_sec_aac.audio",
-        "format_MP4_AVC720P_AAC.video",
+        "sine_wave_1k_44100_stereo_2_sec_mp3.audio",
+        "sine_wave_1k_48000_stereo_2_sec_wav.audio",
     ],
     visibility = ["//visibility:public"],
 )

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -0,0 +1,323 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines TimeSeriesFramerCalculator.
+#include <math.h>
+
+#include <deque>
+#include <memory>
+#include <string>
+
+#include "Eigen/Core"
+#include "audio/dsp/window_functions.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// MediaPipe Calculator for framing a (vector-valued) input time series,
+// i.e. for breaking an input time series into fixed-size, possibly
+// overlapping, frames.  The output stream's frame duration is
+// specified by frame_duration_seconds in the
+// TimeSeriesFramerCalculatorOptions, and the output's overlap is
+// specified by frame_overlap_seconds.
+//
+// This calculator assumes that the input timestamps refer to the
+// first sample in each Matrix.  The output timestamps follow this
+// same convention.
+//
+// All output frames will have exactly the same number of samples: the number of
+// samples that approximates frame_duration_seconds most closely.
+//
+// Similarly, frame overlap is by default the (fixed) number of samples
+// approximating frame_overlap_seconds most closely.  But if
+// emulate_fractional_frame_overlap is set to true, frame overlap is a variable
+// number of samples instead, such that the long-term average step between
+// frames is the difference between the (nominal) frame_duration_seconds and
+// frame_overlap_seconds.
+//
+// If pad_final_packet is true, all input samples will be emitted and the final
+// packet will be zero padded as necessary.  If pad_final_packet is false, some
+// samples may be dropped at the end of the stream.
+//
+// If use_local_timestamp is true, the output packet's timestamp is based on the
+// last sample of the packet. The timestamp of this sample is inferred by
+// input_packet_timesamp + local_sample_index / sampling_rate_. If false, the
+// output packet's timestamp is based on the cumulative timestamping, which is
+// done by adopting the timestamp of the first sample of the packet and this
+// sample's timestamp is inferred by initial_input_timestamp_ +
+// cumulative_completed_samples / sample_rate_.
+class TimeSeriesFramerCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input stream with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Fixed length time series Packets with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  // Returns FAIL if the input stream header is invalid.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Outputs as many framed packets as possible given the accumulated
+  // input.  Always returns OK.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+  // Flushes any remaining samples in a zero-padded packet.  Always
+  // returns OK.
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  // Adds input data to the internal buffer.
+  void EnqueueInput(CalculatorContext* cc);
+  // Constructs and emits framed output packets.
+  void FrameOutput(CalculatorContext* cc);
+
+  Timestamp CurrentOutputTimestamp() {
+    if (use_local_timestamp_) {
+      return current_timestamp_;
+    }
+    return CumulativeOutputTimestamp();
+  }
+
+  Timestamp CumulativeOutputTimestamp() {
+    return initial_input_timestamp_ +
+           round(cumulative_completed_samples_ / sample_rate_ *
+                 Timestamp::kTimestampUnitsPerSecond);
+  }
+
+  // Returns the timestamp of a sample on a base, which is usually the time
+  // stamp of a packet.
+  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
+                                   int64 number_of_samples) {
+    return timestamp_base + round(number_of_samples / sample_rate_ *
+                                  Timestamp::kTimestampUnitsPerSecond);
+  }
+
+  // The number of input samples to advance after the current output frame is
+  // emitted.
+  int next_frame_step_samples() const {
+    // All numbers are in input samples.
+    const int64 current_output_frame_start = static_cast<int64>(
+        round(cumulative_output_frames_ * average_frame_step_samples_));
+    CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
+    const int64 next_output_frame_start = static_cast<int64>(
+        round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
+    return next_output_frame_start - current_output_frame_start;
+  }
+
+  double sample_rate_;
+  bool pad_final_packet_;
+  int frame_duration_samples_;
+  // The advance, in input samples, between the start of successive output
+  // frames. This may be a non-integer average value if
+  // emulate_fractional_frame_overlap is true.
+  double average_frame_step_samples_;
+  int samples_still_to_drop_;
+  int64 cumulative_input_samples_;
+  int64 cumulative_output_frames_;
+  // "Completed" samples are samples that are no longer needed because
+  // the framer has completely stepped past them (taking into account
+  // any overlap).
+  int64 cumulative_completed_samples_;
+  Timestamp initial_input_timestamp_;
+  // The current timestamp is updated along with the incoming packets.
+  Timestamp current_timestamp_;
+  int num_channels_;
+
+  // Each entry in this deque consists of a single sample, i.e. a
+  // single column vector, and its timestamp.
+  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
+
+  bool use_window_;
+  Matrix window_;
+
+  bool use_local_timestamp_;
+};
+REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
+
+void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
+  const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
+
+  for (int i = 0; i < input_frame.cols(); ++i) {
+    sample_buffer_.emplace_back(std::make_pair(
+        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
+  }
+
+  cumulative_input_samples_ += input_frame.cols();
+}
+
+void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
+  while (sample_buffer_.size() >=
+         frame_duration_samples_ + samples_still_to_drop_) {
+    while (samples_still_to_drop_ > 0) {
+      sample_buffer_.pop_front();
+      --samples_still_to_drop_;
+    }
+    const int frame_step_samples = next_frame_step_samples();
+    std::unique_ptr<Matrix> output_frame(
+        new Matrix(num_channels_, frame_duration_samples_));
+    for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
+         ++i) {
+      output_frame->col(i) = sample_buffer_.front().first;
+      current_timestamp_ = sample_buffer_.front().second;
+      sample_buffer_.pop_front();
+    }
+    const int frame_overlap_samples =
+        frame_duration_samples_ - frame_step_samples;
+    if (frame_overlap_samples > 0) {
+      for (int i = 0; i < frame_overlap_samples; ++i) {
+        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
+        current_timestamp_ = sample_buffer_[i].second;
+      }
+    } else {
+      samples_still_to_drop_ = -frame_overlap_samples;
+    }
+
+    if (use_window_) {
+      *output_frame = (output_frame->array() * window_.array()).matrix();
+    }
+
+    cc->Outputs().Index(0).Add(output_frame.release(),
+                               CurrentOutputTimestamp());
+    ++cumulative_output_frames_;
+    cumulative_completed_samples_ += frame_step_samples;
+  }
+}
+
+::mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
+  }
+
+  EnqueueInput(cc);
+  FrameOutput(cc);
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
+  while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
+    sample_buffer_.pop_front();
+    --samples_still_to_drop_;
+  }
+  if (!sample_buffer_.empty() && pad_final_packet_) {
+    std::unique_ptr<Matrix> output_frame(new Matrix);
+    output_frame->setZero(num_channels_, frame_duration_samples_);
+    for (int i = 0; i < sample_buffer_.size(); ++i) {
+      output_frame->col(i) = sample_buffer_[i].first;
+      current_timestamp_ = sample_buffer_[i].second;
+    }
+
+    cc->Outputs().Index(0).Add(output_frame.release(),
+                               CurrentOutputTimestamp());
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
+  TimeSeriesFramerCalculatorOptions framer_options =
+      cc->Options<TimeSeriesFramerCalculatorOptions>();
+
+  RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0)
+      << "Invalid or missing frame_duration_seconds. "
+      << "framer_duration_seconds: \n"
+      << framer_options.frame_duration_seconds();
+  RET_CHECK_LT(framer_options.frame_overlap_seconds(),
+               framer_options.frame_duration_seconds())
+      << "Invalid frame_overlap_seconds. framer_overlap_seconds: \n"
+      << framer_options.frame_overlap_seconds();
+
+  TimeSeriesHeader input_header;
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+      cc->Inputs().Index(0).Header(), &input_header));
+
+  sample_rate_ = input_header.sample_rate();
+  num_channels_ = input_header.num_channels();
+  frame_duration_samples_ = time_series_util::SecondsToSamples(
+      framer_options.frame_duration_seconds(), sample_rate_);
+  RET_CHECK_GT(frame_duration_samples_, 0)
+      << "Frame duration of " << framer_options.frame_duration_seconds()
+      << "s too small to cover a single sample at " << sample_rate_ << " Hz ";
+  if (framer_options.emulate_fractional_frame_overlap()) {
+    // Frame step may be fractional.
+    average_frame_step_samples_ = (framer_options.frame_duration_seconds() -
+                                   framer_options.frame_overlap_seconds()) *
+                                  sample_rate_;
+  } else {
+    // Frame step is an integer (stored in a double).
+    average_frame_step_samples_ =
+        frame_duration_samples_ -
+        time_series_util::SecondsToSamples(
+            framer_options.frame_overlap_seconds(), sample_rate_);
+  }
+  RET_CHECK_GE(average_frame_step_samples_, 1)
+      << "Frame step too small to cover a single sample at " << sample_rate_
+      << " Hz.";
+  pad_final_packet_ = framer_options.pad_final_packet();
+
+  auto output_header = new TimeSeriesHeader(input_header);
+  output_header->set_num_samples(frame_duration_samples_);
+  if (round(average_frame_step_samples_) == average_frame_step_samples_) {
+    // Only set output packet rate if it is fixed.
+    output_header->set_packet_rate(sample_rate_ / average_frame_step_samples_);
+  }
+  cc->Outputs().Index(0).SetHeader(Adopt(output_header));
+  cumulative_completed_samples_ = 0;
+  cumulative_input_samples_ = 0;
+  cumulative_output_frames_ = 0;
+  samples_still_to_drop_ = 0;
+  initial_input_timestamp_ = Timestamp::Unstarted();
+  current_timestamp_ = Timestamp::Unstarted();
+
+  std::vector<double> window_vector;
+  use_window_ = false;
+  switch (framer_options.window_function()) {
+    case TimeSeriesFramerCalculatorOptions::HAMMING:
+      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                    &window_vector);
+      use_window_ = true;
+      break;
+    case TimeSeriesFramerCalculatorOptions::HANN:
+      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                 &window_vector);
+      use_window_ = true;
+      break;
+    case TimeSeriesFramerCalculatorOptions::NONE:
+      break;
+  }
+
+  if (use_window_) {
+    window_ = Matrix::Ones(num_channels_, 1) *
+              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
+                                          frame_duration_samples_)
+                  .cast<float>();
+  }
+  use_local_timestamp_ = framer_options.use_local_timestamp();
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/time_series_framer_calculator.proto

@@ -0,0 +1,72 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message TimeSeriesFramerCalculatorOptions {
+  extend CalculatorOptions {
+    optional TimeSeriesFramerCalculatorOptions ext = 50631621;
+  }
+
+  // Frame duration in seconds.  Required.  Must be greater than 0.  This is
+  // rounded to the nearest integer number of samples.
+  optional double frame_duration_seconds = 1;
+
+  // Frame overlap in seconds.
+  //
+  // If emulate_fractional_frame_overlap is false (the default), then the frame
+  // overlap is rounded to the nearest integer number of samples, and the step
+  // from one frame to the next will be the difference between the number of
+  // samples in a frame and the number of samples in the overlap.
+  //
+  // If emulate_fractional_frame_overlap is true, then frame overlap will be a
+  // variable number of samples, such that the long-time average time step from
+  // one frame to the next will be the difference between the (nominal, not
+  // rounded) frame_duration_seconds and frame_overlap_seconds.  This is useful
+  // where the desired time step is not an integral number of input samples.
+  //
+  // A negative frame_overlap_seconds corresponds to skipping some input samples
+  // between each frame of emitted samples.
+  //
+  // Required that frame_overlap_seconds <  frame_duration_seconds.
+  optional double frame_overlap_seconds = 2 [default = 0.0];
+
+  // See frame_overlap_seconds for semantics.
+  optional bool emulate_fractional_frame_overlap = 5 [default = false];
+
+  // Whether to pad the final packet with zeros.  If true, guarantees that all
+  // input samples (other than those that fall in gaps implied by negative
+  // frame_overlap_seconds) will be emitted.  If set to false, any partial
+  // packet at the end of the stream will be dropped.
+  optional bool pad_final_packet = 3 [default = true];
+
+  // Optional windowing function. The default is NONE (no windowing function).
+  enum WindowFunction {
+    NONE = 0;
+    HAMMING = 1;
+    HANN = 2;
+  }
+  optional WindowFunction window_function = 4 [default = NONE];
+
+  // If use_local_timestamp is true, the output packet's timestamp is based on
+  // the last sample of the packet and it's inferred from the latest input
+  // packet's timestamp.  If false, the output packet's timestamp is based on
+  // the cumulative timestamping, which is inferred from the intial input
+  // timestamp and the cumulative number of samples.
+  optional bool use_local_timestamp = 6 [default = false];
+}

mediapipe/calculators/audio/time_series_framer_calculator_test.cc

@@ -0,0 +1,485 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Eigen/Core"
+#include "audio/dsp/window_functions.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+namespace {
+
+const int kInitialTimestampOffsetMicroseconds = 4;
+const int kGapBetweenPacketsInSeconds = 1;
+const int kUniversalInputPacketSize = 50;
+
+class TimeSeriesFramerCalculatorTest
+    : public TimeSeriesCalculatorTest<TimeSeriesFramerCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "TimeSeriesFramerCalculator";
+    input_sample_rate_ = 4000.0;
+    num_input_channels_ = 3;
+  }
+
+  // Returns a float value with the channel and timestamp separated by
+  // an order of magnitude, for easy parsing by humans.
+  float TestValue(int64 timestamp_in_microseconds, int channel) {
+    return timestamp_in_microseconds + channel / 10.0;
+  }
+
+  // Caller takes ownership of the returned value.
+  Matrix* NewTestFrame(int num_channels, int num_samples,
+                       double starting_timestamp_seconds) {
+    auto matrix = new Matrix(num_channels, num_samples);
+    for (int c = 0; c < num_channels; ++c) {
+      for (int i = 0; i < num_samples; ++i) {
+        int64 timestamp = time_series_util::SecondsToSamples(
+            starting_timestamp_seconds + i / input_sample_rate_,
+            Timestamp::kTimestampUnitsPerSecond);
+        (*matrix)(c, i) = TestValue(timestamp, c);
+      }
+    }
+    return matrix;
+  }
+
+  // Initializes and runs the test graph.
+  ::mediapipe::Status Run() {
+    InitializeGraph();
+
+    FillInputHeader();
+    InitializeInput();
+
+    return RunGraph();
+  }
+
+  // Creates test input and saves a reference copy.
+  void InitializeInput() {
+    concatenated_input_samples_.resize(0, num_input_channels_);
+    num_input_samples_ = 0;
+    for (int i = 0; i < 10; ++i) {
+      // This range of packet sizes was chosen such that some input
+      // packets will be smaller than the output packet size and other
+      // input packets will be larger.
+      int packet_size = (i + 1) * 20;
+      double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+                                 num_input_samples_ / input_sample_rate_;
+
+      Matrix* data_frame =
+          NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
+
+      // Keep a reference copy of the input.
+      //
+      // conservativeResize() is needed here to preserve the existing
+      // data.  Eigen's resize() resizes without preserving data.
+      concatenated_input_samples_.conservativeResize(
+          num_input_channels_, num_input_samples_ + packet_size);
+      concatenated_input_samples_.rightCols(packet_size) = *data_frame;
+      num_input_samples_ += packet_size;
+
+      AppendInputPacket(data_frame, round(timestamp_seconds *
+                                          Timestamp::kTimestampUnitsPerSecond));
+    }
+
+    const int frame_duration_samples = FrameDurationSamples();
+    std::vector<double> window_vector;
+    switch (options_.window_function()) {
+      case TimeSeriesFramerCalculatorOptions::HAMMING:
+        audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples,
+                                                      &window_vector);
+        break;
+      case TimeSeriesFramerCalculatorOptions::HANN:
+        audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples,
+                                                   &window_vector);
+        break;
+      case TimeSeriesFramerCalculatorOptions::NONE:
+        window_vector.assign(frame_duration_samples, 1.0f);
+        break;
+    }
+
+    window_ = Matrix::Ones(num_input_channels_, 1) *
+              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
+                                          frame_duration_samples)
+                  .cast<float>();
+  }
+
+  int FrameDurationSamples() {
+    return time_series_util::SecondsToSamples(options_.frame_duration_seconds(),
+                                              input_sample_rate_);
+  }
+
+  // Checks that the values in the framed output packets matches the
+  // appropriate values from the input.
+  void CheckOutputPacketValues(const Matrix& actual, int packet_num,
+                               int frame_duration_samples,
+                               double frame_step_samples,
+                               int num_columns_to_check) {
+    ASSERT_EQ(frame_duration_samples, actual.cols());
+    Matrix expected = (concatenated_input_samples_
+                           .block(0, round(frame_step_samples * packet_num),
+                                  num_input_channels_, num_columns_to_check)
+                           .array() *
+                       window_.leftCols(num_columns_to_check).array())
+                          .matrix();
+    ExpectApproximatelyEqual(expected, actual.leftCols(num_columns_to_check));
+  }
+
+  // Checks output headers, Timestamps, and values.
+  void CheckOutput() {
+    const int frame_duration_samples = FrameDurationSamples();
+    const double frame_step_samples =
+        options_.emulate_fractional_frame_overlap()
+            ? (options_.frame_duration_seconds() -
+               options_.frame_overlap_seconds()) *
+                  input_sample_rate_
+            : frame_duration_samples -
+                  time_series_util::SecondsToSamples(
+                      options_.frame_overlap_seconds(), input_sample_rate_);
+
+    TimeSeriesHeader expected_header = input().header.Get<TimeSeriesHeader>();
+    expected_header.set_num_samples(frame_duration_samples);
+    if (!options_.emulate_fractional_frame_overlap() ||
+        frame_step_samples == round(frame_step_samples)) {
+      expected_header.set_packet_rate(input_sample_rate_ / frame_step_samples);
+    }
+    ExpectOutputHeaderEquals(expected_header);
+
+    int num_full_packets = output().packets.size();
+    if (options_.pad_final_packet()) {
+      num_full_packets -= 1;
+    }
+
+    for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
+      const Packet& packet = output().packets[packet_num];
+      CheckOutputPacketValues(packet.Get<Matrix>(), packet_num,
+                              frame_duration_samples, frame_step_samples,
+                              frame_duration_samples);
+    }
+
+    // What is the effective time index of the final sample emitted?
+    // This includes accounting for the gaps when overlap is negative.
+    const int num_unique_output_samples =
+        round((output().packets.size() - 1) * frame_step_samples) +
+        frame_duration_samples;
+    LOG(INFO) << "packets.size()=" << output().packets.size()
+              << " frame_duration_samples=" << frame_duration_samples
+              << " frame_step_samples=" << frame_step_samples
+              << " num_input_samples_=" << num_input_samples_
+              << " num_unique_output_samples=" << num_unique_output_samples;
+    const int num_padding_samples =
+        num_unique_output_samples - num_input_samples_;
+    if (options_.pad_final_packet()) {
+      EXPECT_LT(num_padding_samples, frame_duration_samples);
+      // If the input ended during the dropped samples between the end of
+      // the last emitted frame and where the next one would begin, there
+      // can be fewer unique output points than input points, even with
+      // padding.
+      const int max_dropped_samples =
+          static_cast<int>(ceil(frame_step_samples - frame_duration_samples));
+      EXPECT_GE(num_padding_samples, std::min(0, -max_dropped_samples));
+
+      if (num_padding_samples > 0) {
+        // Check the non-padded part of the final packet.
+        const Matrix& final_matrix = output().packets.back().Get<Matrix>();
+        CheckOutputPacketValues(final_matrix, num_full_packets,
+                                frame_duration_samples, frame_step_samples,
+                                frame_duration_samples - num_padding_samples);
+        // Check the padded part of the final packet.
+        EXPECT_EQ(
+            Matrix::Zero(num_input_channels_, num_padding_samples),
+            final_matrix.block(0, frame_duration_samples - num_padding_samples,
+                               num_input_channels_, num_padding_samples));
+      }
+    } else {
+      EXPECT_GT(num_padding_samples, -frame_duration_samples);
+      EXPECT_LE(num_padding_samples, 0);
+    }
+  }
+
+  int num_input_samples_;
+  Matrix concatenated_input_samples_;
+  Matrix window_;
+};
+
+TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationNoOverlap) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest,
+       IntegerSampleDurationNoOverlapHammingWindow) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING);
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest,
+       IntegerSampleDurationNoOverlapHannWindow) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN);
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationAndOverlap) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(40.0 / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, NonintegerSampleDurationAndOverlap) {
+  options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(38.4 / input_sample_rate_);
+
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFrames) {
+  // Negative overlap means to drop samples between frames.
+  // 100 samples per frame plus a skip of 10 samples will be 10 full frames in
+  // the 1100 input samples.
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(-10.0 / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 10);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFramesLessSkip) {
+  // 100 samples per frame plus a skip of 100 samples will be 6 full frames in
+  // the 1100 input samples.
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 6);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapWithPadding) {
+  // 150 samples per frame plus a skip of 50 samples will require some padding
+  // on the sixth and last frame given 1100 sample input.
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 6);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, FixedFrameOverlap) {
+  // Frame of 30 samples with step of 11.4 samples (rounded to 11 samples)
+  // results in ceil((1100 - 30) / 11) + 1 = 99 packets.
+  options_.set_frame_duration_seconds(30 / input_sample_rate_);
+  options_.set_frame_overlap_seconds((30.0 - 11.4) / input_sample_rate_);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 99);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameOverlap) {
+  // Frame of 30 samples with step of 11.4 samples (not rounded)
+  // results in ceil((1100 - 30) / 11.4) + 1 = 95 packets.
+  options_.set_frame_duration_seconds(30 / input_sample_rate_);
+  options_.set_frame_overlap_seconds((30 - 11.4) / input_sample_rate_);
+  options_.set_emulate_fractional_frame_overlap(true);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 95);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameSkip) {
+  // Frame of 30 samples with step of 41.4 samples (not rounded)
+  // results in ceil((1100 - 30) / 41.4) + 1 = 27 packets.
+  options_.set_frame_duration_seconds(30 / input_sample_rate_);
+  options_.set_frame_overlap_seconds((30 - 41.4) / input_sample_rate_);
+  options_.set_emulate_fractional_frame_overlap(true);
+  MP_ASSERT_OK(Run());
+  EXPECT_EQ(output().packets.size(), 27);
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest, NoFinalPacketPadding) {
+  options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
+  options_.set_pad_final_packet(false);
+
+  MP_ASSERT_OK(Run());
+  CheckOutput();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest,
+       FrameRateHigherThanSampleRate_FrameDurationTooLow) {
+  // Try to produce a frame rate 10 times the input sample rate by using a
+  // a frame duration that is too small and covers only 0.1 samples.
+  options_.set_frame_duration_seconds(1 / (10 * input_sample_rate_));
+  options_.set_frame_overlap_seconds(0.0);
+  EXPECT_FALSE(Run().ok());
+}
+
+TEST_F(TimeSeriesFramerCalculatorTest,
+       FrameRateHigherThanSampleRate_FrameStepTooLow) {
+  // Try to produce a frame rate 10 times the input sample rate by using
+  // a frame overlap that is too high and produces frame steps (difference
+  // between duration and overlap) of 0.1 samples.
+  options_.set_frame_duration_seconds(10.0 / input_sample_rate_);
+  options_.set_frame_overlap_seconds(9.9 / input_sample_rate_);
+  EXPECT_FALSE(Run().ok());
+}
+
+// A simple test class to do windowing sanity checks. Tests from this
+// class input a single packet of all ones, and check the average
+// value of the single output packet. This is useful as a sanity check
+// that the correct windows are applied.
+class TimeSeriesFramerCalculatorWindowingSanityTest
+    : public TimeSeriesFramerCalculatorTest {
+ protected:
+  void SetUp() override {
+    TimeSeriesFramerCalculatorTest::SetUp();
+    num_input_channels_ = 1;
+  }
+
+  void RunAndTestSinglePacketAverage(float expected_average) {
+    options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+    InitializeGraph();
+    FillInputHeader();
+    AppendInputPacket(new Matrix(Matrix::Ones(1, FrameDurationSamples())),
+                      kInitialTimestampOffsetMicroseconds);
+    MP_ASSERT_OK(RunGraph());
+    ASSERT_EQ(1, output().packets.size());
+    ASSERT_NEAR(expected_average * FrameDurationSamples(),
+                output().packets[0].Get<Matrix>().sum(), 1e-5);
+  }
+};
+
+TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, NoWindowSanityCheck) {
+  RunAndTestSinglePacketAverage(1.0f);
+}
+
+TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest,
+       HammingWindowSanityCheck) {
+  options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING);
+  RunAndTestSinglePacketAverage(0.54f);
+}
+
+TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, HannWindowSanityCheck) {
+  options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN);
+  RunAndTestSinglePacketAverage(0.5f);
+}
+
+// A simple test class that checks the local packet time stamp. This class
+// generate a series of packets with and without gaps between packets and tests
+// the behavior with cumulative timestamping and local packet timestamping.
+class TimeSeriesFramerCalculatorTimestampingTest
+    : public TimeSeriesFramerCalculatorTest {
+ protected:
+  // Creates test input and saves a reference copy.
+  void InitializeInputForTimeStampingTest() {
+    concatenated_input_samples_.resize(0, num_input_channels_);
+    num_input_samples_ = 0;
+    for (int i = 0; i < 10; ++i) {
+      // This range of packet sizes was chosen such that some input
+      // packets will be smaller than the output packet size and other
+      // input packets will be larger.
+      int packet_size = kUniversalInputPacketSize;
+      double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+                                 num_input_samples_ / input_sample_rate_;
+      if (options_.use_local_timestamp()) {
+        timestamp_seconds += kGapBetweenPacketsInSeconds * i;
+      }
+
+      Matrix* data_frame =
+          NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
+
+      AppendInputPacket(data_frame, round(timestamp_seconds *
+                                          Timestamp::kTimestampUnitsPerSecond));
+      num_input_samples_ += packet_size;
+    }
+  }
+
+  void CheckOutputTimestamps() {
+    int num_full_packets = output().packets.size();
+    if (options_.pad_final_packet()) {
+      num_full_packets -= 1;
+    }
+
+    int64 num_samples = 0;
+    for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
+      const Packet& packet = output().packets[packet_num];
+      num_samples += FrameDurationSamples();
+      double expected_timestamp =
+          options_.use_local_timestamp()
+              ? GetExpectedLocalTimestampForSample(num_samples - 1)
+              : GetExpectedCumulativeTimestamp(num_samples - 1);
+      ASSERT_NEAR(packet.Timestamp().Seconds(), expected_timestamp, 1e-10);
+    }
+  }
+
+  ::mediapipe::Status RunTimestampTest() {
+    InitializeGraph();
+    InitializeInputForTimeStampingTest();
+    FillInputHeader();
+    return RunGraph();
+  }
+
+ private:
+  // Returns the timestamp in seconds based on local timestamping.
+  double GetExpectedLocalTimestampForSample(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / input_sample_rate_ +
+           (sample_index / kUniversalInputPacketSize) *
+               kGapBetweenPacketsInSeconds;
+  }
+
+  // Returns the timestamp inseconds based on cumulative timestamping.
+  double GetExpectedCumulativeTimestamp(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / FrameDurationSamples() * FrameDurationSamples() /
+               input_sample_rate_;
+  }
+};
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseLocalTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(true);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseCumulativeTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(false);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/BUILD

@@ -13,11 +13,32 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+proto_library(
+    name = "concatenate_vector_calculator_proto",
+    srcs = ["concatenate_vector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "dequantize_byte_array_calculator_proto",
+    srcs = ["dequantize_byte_array_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "packet_cloner_calculator_proto",
+    srcs = ["packet_cloner_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
 
 proto_library(
     name = "packet_resampler_calculator_proto",
@@ -26,14 +47,297 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "split_vector_calculator_proto",
+    srcs = ["split_vector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "quantize_float_vector_calculator_proto",
+    srcs = ["quantize_float_vector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "sequence_shift_calculator_proto",
+    srcs = ["sequence_shift_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+proto_library(
+    name = "gate_calculator_proto",
+    srcs = ["gate_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+proto_library(
+    name = "clip_vector_size_calculator_proto",
+    srcs = ["clip_vector_size_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "packet_cloner_calculator_cc_proto",
+    srcs = ["packet_cloner_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":packet_cloner_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "packet_resampler_calculator_cc_proto",
     srcs = ["packet_resampler_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":packet_resampler_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "split_vector_calculator_cc_proto",
+    srcs = ["split_vector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":split_vector_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "concatenate_vector_calculator_cc_proto",
+    srcs = ["concatenate_vector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":concatenate_vector_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "clip_vector_size_calculator_cc_proto",
+    srcs = ["clip_vector_size_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":clip_vector_size_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "dequantize_byte_array_calculator_cc_proto",
+    srcs = ["dequantize_byte_array_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":dequantize_byte_array_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "quantize_float_vector_calculator_cc_proto",
+    srcs = ["quantize_float_vector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":quantize_float_vector_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "sequence_shift_calculator_cc_proto",
+    srcs = ["sequence_shift_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":sequence_shift_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "gate_calculator_cc_proto",
+    srcs = ["gate_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":gate_calculator_proto"],
+)
+
+cc_library(
+    name = "add_header_calculator",
+    srcs = ["add_header_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:logging",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "add_header_calculator_test",
+    size = "small",
+    srcs = ["add_header_calculator_test.cc"],
+    deps = [
+        ":add_header_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
+cc_library(
+    name = "begin_loop_calculator",
+    srcs = ["begin_loop_calculator.cc"],
+    hdrs = ["begin_loop_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "end_loop_calculator",
+    srcs = ["end_loop_calculator.cc"],
+    hdrs = ["end_loop_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:render_data_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "begin_end_loop_calculator_graph_test",
+    srcs = ["begin_end_loop_calculator_graph_test.cc"],
+    deps = [
+        ":begin_loop_calculator",
+        ":end_loop_calculator",
+        "//mediapipe/calculators/core:packet_cloner_calculator",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
+cc_library(
+    name = "concatenate_vector_calculator",
+    srcs = ["concatenate_vector_calculator.cc"],
+    hdrs = ["concatenate_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "concatenate_detection_vector_calculator",
+    srcs = ["concatenate_detection_vector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "concatenate_vector_calculator_test",
+    srcs = ["concatenate_vector_calculator_test.cc"],
+    deps = [
+        ":concatenate_vector_calculator",
+        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "clip_vector_size_calculator",
+    srcs = ["clip_vector_size_calculator.cc"],
+    hdrs = ["clip_vector_size_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":clip_vector_size_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "clip_detection_vector_size_calculator",
+    srcs = ["clip_detection_vector_size_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":clip_vector_size_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "clip_vector_size_calculator_test",
+    srcs = ["clip_vector_size_calculator_test.cc"],
+    deps = [
+        ":clip_vector_size_calculator",
+        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "counting_source_calculator",
     srcs = ["counting_source_calculator.cc"],
@@ -62,6 +366,38 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "matrix_multiply_calculator",
+    srcs = ["matrix_multiply_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:status",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "matrix_subtract_calculator",
+    srcs = ["matrix_subtract_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:status",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "mux_calculator",
     srcs = ["mux_calculator.cc"],
@@ -83,12 +419,37 @@ cc_library(
         "//visibility:public",
     ],
     deps = [
+        ":packet_cloner_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
 )
 
+cc_library(
+    name = "packet_inner_join_calculator",
+    srcs = ["packet_inner_join_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "packet_inner_join_calculator_test",
+    srcs = ["packet_inner_join_calculator_test.cc"],
+    deps = [
+        ":packet_inner_join_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
 cc_library(
     name = "pass_through_calculator",
     srcs = ["pass_through_calculator.cc"],
@@ -145,8 +506,8 @@ cc_library(
 )
 
 cc_library(
-    name = "real_time_flow_limiter_calculator",
+    name = "flow_limiter_calculator",
-    srcs = ["real_time_flow_limiter_calculator.cc"],
+    srcs = ["flow_limiter_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
@@ -160,6 +521,32 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "string_to_int_calculator",
+    srcs = ["string_to_int_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "side_packet_to_stream_calculator",
+    srcs = ["side_packet_to_stream_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "immediate_mux_calculator_test",
     srcs = ["immediate_mux_calculator_test.cc"],
@@ -191,19 +578,16 @@ cc_library(
         "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework/deps:mathutil",
+        "//mediapipe/framework/deps:random",
         "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:options_util",
         "@com_google_absl//absl/strings",
-        "//mediapipe/framework/deps:mathutil",
+    ],
-        "//mediapipe/framework/port:status",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:integral_types",
-    ] + select({
-        "//conditions:default": [
-            "//mediapipe/framework/deps:random",
-        ],
-    }),
     alwayslink = 1,
 )
 
@@ -245,10 +629,43 @@ cc_test(
 )
 
 cc_test(
-    name = "real_time_flow_limiter_calculator_test",
+    name = "matrix_multiply_calculator_test",
-    srcs = ["real_time_flow_limiter_calculator_test.cc"],
+    srcs = ["matrix_multiply_calculator_test.cc"],
+    visibility = ["//visibility:private"],
     deps = [
-        ":real_time_flow_limiter_calculator",
+        ":matrix_multiply_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/tool:validate_type",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "matrix_subtract_calculator_test",
+    srcs = ["matrix_subtract_calculator_test.cc"],
+    visibility = ["//visibility:private"],
+    deps = [
+        ":matrix_subtract_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:validate_type",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+    name = "flow_limiter_calculator_test",
+    srcs = ["flow_limiter_calculator_test.cc"],
+    deps = [
+        ":flow_limiter_calculator",
         "//mediapipe/calculators/core:counting_source_calculator",
         "//mediapipe/calculators/core:pass_through_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -264,3 +681,228 @@ cc_test(
         "@com_google_absl//absl/time",
     ],
 )
+
+cc_library(
+    name = "split_vector_calculator",
+    srcs = ["split_vector_calculator.cc"],
+    hdrs = ["split_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+        "@org_tensorflow//tensorflow/lite:framework",
+        "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "split_vector_calculator_test",
+    srcs = ["split_vector_calculator_test.cc"],
+    deps = [
+        ":split_vector_calculator",
+        ":split_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:validate_type",
+        "@org_tensorflow//tensorflow/lite:framework",
+        "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
+    ],
+)
+
+cc_library(
+    name = "dequantize_byte_array_calculator",
+    srcs = ["dequantize_byte_array_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":dequantize_byte_array_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "dequantize_byte_array_calculator_test",
+    srcs = ["dequantize_byte_array_calculator_test.cc"],
+    deps = [
+        ":dequantize_byte_array_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "quantize_float_vector_calculator",
+    srcs = ["quantize_float_vector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":quantize_float_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "quantize_float_vector_calculator_test",
+    srcs = ["quantize_float_vector_calculator_test.cc"],
+    deps = [
+        ":quantize_float_vector_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "sequence_shift_calculator",
+    srcs = ["sequence_shift_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":sequence_shift_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "sequence_shift_calculator_test",
+    srcs = ["sequence_shift_calculator_test.cc"],
+    deps = [
+        ":sequence_shift_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+    ],
+)
+
+cc_library(
+    name = "gate_calculator",
+    srcs = ["gate_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":gate_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:header_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "gate_calculator_test",
+    srcs = ["gate_calculator_test.cc"],
+    deps = [
+        ":gate_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "matrix_to_vector_calculator",
+    srcs = ["matrix_to_vector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:status_util",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "matrix_to_vector_calculator_test",
+    srcs = ["matrix_to_vector_calculator_test.cc"],
+    deps = [
+        ":matrix_to_vector_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+        "//mediapipe/util:time_series_test_util",
+        "//mediapipe/util:time_series_util",
+    ],
+)
+
+cc_library(
+    name = "merge_calculator",
+    srcs = ["merge_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "merge_calculator_test",
+    srcs = ["merge_calculator_test.cc"],
+    deps = [
+        ":merge_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+cc_library(
+    name = "stream_to_side_packet_calculator",
+    srcs = ["stream_to_side_packet_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "stream_to_side_packet_calculator_test",
+    srcs = ["stream_to_side_packet_calculator_test.cc"],
+    deps = [
+        ":stream_to_side_packet_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)

mediapipe/calculators/core/add_header_calculator.cc

@@ -0,0 +1,53 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/logging.h"
+
+namespace mediapipe {
+
+// Attach the header from one stream to another stream.
+//
+// The header stream (tag HEADER) must not have any packets in it.
+//
+// Before using this calculator, please think about changing your
+// calculator to not need a header or to accept a separate stream with
+// a header, that would be more future proof.
+//
+class AddHeaderCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag("HEADER").SetNone();
+    cc->Inputs().Tag("DATA").SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Tag("DATA"));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const Packet& header = cc->Inputs().Tag("HEADER").Header();
+    if (!header.IsEmpty()) {
+      cc->Outputs().Index(0).SetHeader(header);
+    }
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    cc->Outputs().Index(0).AddPacket(cc->Inputs().Tag("DATA").Value());
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(AddHeaderCalculator);
+}  // namespace mediapipe

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -0,0 +1,99 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+class AddHeaderCalculatorTest : public ::testing::Test {};
+
+TEST_F(AddHeaderCalculatorTest, Works) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("HEADER:header_stream");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+
+  CalculatorRunner runner(node);
+
+  // Set header and add 5 packets.
+  runner.MutableInputs()->Tag("HEADER").header =
+      Adopt(new std::string("my_header"));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run calculator.
+  MP_ASSERT_OK(runner.Run());
+
+  ASSERT_EQ(1, runner.Outputs().NumEntries());
+
+  // Test output.
+  EXPECT_EQ(std::string("my_header"),
+            runner.Outputs().Index(0).header.Get<std::string>());
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(5, output_packets.size());
+  for (int i = 0; i < 5; ++i) {
+    const int val = output_packets[i].Get<int>();
+    EXPECT_EQ(i, val);
+    EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp());
+  }
+}
+
+TEST_F(AddHeaderCalculatorTest, HandlesEmptyHeaderStream) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("HEADER:header_stream");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+
+  CalculatorRunner runner(node);
+
+  // No header and no packets.
+  // Run calculator.
+  MP_ASSERT_OK(runner.Run());
+  EXPECT_TRUE(runner.Outputs().Index(0).header.IsEmpty());
+}
+
+TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) {
+  CalculatorGraphConfig::Node node;
+  node.set_calculator("AddHeaderCalculator");
+  node.add_input_stream("HEADER:header_stream");
+  node.add_input_stream("DATA:data_stream");
+  node.add_output_stream("merged_stream");
+
+  CalculatorRunner runner(node);
+
+  // Set header and add 5 packets.
+  runner.MutableInputs()->Tag("HEADER").header =
+      Adopt(new std::string("my_header"));
+  runner.MutableInputs()->Tag("HEADER").packets.push_back(
+      Adopt(new std::string("not allowed")));
+  for (int i = 0; i < 5; ++i) {
+    Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000));
+    runner.MutableInputs()->Tag("DATA").packets.push_back(packet);
+  }
+
+  // Run calculator.
+  ASSERT_FALSE(runner.Run().ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -0,0 +1,335 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/core/begin_loop_calculator.h"
+#include "mediapipe/calculators/core/end_loop_calculator.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+namespace {
+
+typedef BeginLoopCalculator<std::vector<int>> BeginLoopIntegerCalculator;
+REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
+
+class IncrementCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const int& input_int = cc->Inputs().Index(0).Get<int>();
+    auto output_int = absl::make_unique<int>(input_int + 1);
+    cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(IncrementCalculator);
+
+typedef EndLoopCalculator<std::vector<int>> EndLoopIntegersCalculator;
+REGISTER_CALCULATOR(EndLoopIntegersCalculator);
+
+class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
+ protected:
+  BeginEndLoopCalculatorGraphTest() {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        R"(
+          num_threads: 4
+          input_stream: "ints"
+          node {
+            calculator: "BeginLoopIntegerCalculator"
+            input_stream: "ITERABLE:ints"
+            output_stream: "ITEM:int"
+            output_stream: "BATCH_END:timestamp"
+          }
+          node {
+            calculator: "IncrementCalculator"
+            input_stream: "int"
+            output_stream: "int_plus_one"
+          }
+          node {
+            calculator: "EndLoopIntegersCalculator"
+            input_stream: "ITEM:int_plus_one"
+            input_stream: "BATCH_END:timestamp"
+            output_stream: "ITERABLE:ints_plus_one"
+          }
+        )");
+    tool::AddVectorSink("ints_plus_one", &graph_config_, &output_packets_);
+  }
+
+  CalculatorGraphConfig graph_config_;
+  std::vector<Packet> output_packets_;
+};
+
+TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // EndLoopCalc will forward the timestamp bound because there are no elements
+  // in collection to output.
+  ASSERT_EQ(0, output_packets_.size());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  input_vector->emplace_back(0);
+  input_vector->emplace_back(1);
+  input_vector->emplace_back(2);
+  Timestamp input_timestamp = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ASSERT_EQ(1, output_packets_.size());
+  EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector = {1, 2, 3};
+  EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  auto input_vector0 = absl::make_unique<std::vector<int>>();
+  input_vector0->emplace_back(0);
+  input_vector0->emplace_back(1);
+  Timestamp input_timestamp0 = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector0.release()).At(input_timestamp0)));
+
+  auto input_vector1 = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp1 = Timestamp(1);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector1.release()).At(input_timestamp1)));
+
+  auto input_vector2 = absl::make_unique<std::vector<int>>();
+  input_vector2->emplace_back(2);
+  input_vector2->emplace_back(3);
+  Timestamp input_timestamp2 = Timestamp(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector2.release()).At(input_timestamp2)));
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ASSERT_EQ(2, output_packets_.size());
+
+  EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector0 = {1, 2};
+  EXPECT_EQ(expected_output_vector0,
+            output_packets_[0].Get<std::vector<int>>());
+
+  // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
+  // no elements in vector to process.
+
+  EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
+  std::vector<int> expected_output_vector2 = {3, 4};
+  EXPECT_EQ(expected_output_vector2,
+            output_packets_[1].Get<std::vector<int>>());
+}
+
+class MultiplierCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Inputs().Index(1).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const int& input_int = cc->Inputs().Index(0).Get<int>();
+    const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
+    auto output_int = absl::make_unique<int>(input_int * multiplier_int);
+    cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(MultiplierCalculator);
+
+class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
+ protected:
+  BeginEndLoopCalculatorGraphWithClonedInputsTest() {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        R"(
+          num_threads: 4
+          input_stream: "ints"
+          input_stream: "multiplier"
+          node {
+            calculator: "BeginLoopIntegerCalculator"
+            input_stream: "ITERABLE:ints"
+            input_stream: "CLONE:multiplier"
+            output_stream: "ITEM:int_at_loop"
+            output_stream: "CLONE:multiplier_cloned_at_loop"
+            output_stream: "BATCH_END:timestamp"
+          }
+          node {
+            calculator: "MultiplierCalculator"
+            input_stream: "int_at_loop"
+            input_stream: "multiplier_cloned_at_loop"
+            output_stream: "multiplied_int_at_loop"
+          }
+          node {
+            calculator: "EndLoopIntegersCalculator"
+            input_stream: "ITEM:multiplied_int_at_loop"
+            input_stream: "BATCH_END:timestamp"
+            output_stream: "ITERABLE:multiplied_ints"
+          }
+        )");
+    tool::AddVectorSink("multiplied_ints", &graph_config_, &output_packets_);
+  }
+
+  CalculatorGraphConfig graph_config_;
+  std::vector<Packet> output_packets_;
+};
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  auto multiplier = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // EndLoopCalc will forward the timestamp bound because there are no elements
+  // in collection to output.
+  ASSERT_EQ(0, output_packets_.size());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  input_vector->emplace_back(0);
+  input_vector->emplace_back(1);
+  input_vector->emplace_back(2);
+  Timestamp input_timestamp = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  auto multiplier = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ASSERT_EQ(1, output_packets_.size());
+  EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector = {0, 2, 4};
+  EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  auto input_vector0 = absl::make_unique<std::vector<int>>();
+  input_vector0->emplace_back(0);
+  input_vector0->emplace_back(1);
+  Timestamp input_timestamp0 = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector0.release()).At(input_timestamp0)));
+  auto multiplier0 = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier0.release()).At(input_timestamp0)));
+
+  auto input_vector1 = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp1 = Timestamp(43);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector1.release()).At(input_timestamp1)));
+  auto multiplier1 = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier1.release()).At(input_timestamp1)));
+
+  auto input_vector2 = absl::make_unique<std::vector<int>>();
+  input_vector2->emplace_back(2);
+  input_vector2->emplace_back(3);
+  Timestamp input_timestamp2 = Timestamp(44);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector2.release()).At(input_timestamp2)));
+  auto multiplier2 = absl::make_unique<int>(3);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier2.release()).At(input_timestamp2)));
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ASSERT_EQ(2, output_packets_.size());
+
+  EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector0 = {0, 2};
+  EXPECT_EQ(expected_output_vector0,
+            output_packets_[0].Get<std::vector<int>>());
+
+  // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
+  // no elements in vector to process.
+
+  EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
+  std::vector<int> expected_output_vector2 = {6, 9};
+  EXPECT_EQ(expected_output_vector2,
+            output_packets_[1].Get<std::vector<int>>());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -0,0 +1,40 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/begin_loop_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+// A calculator to process std::vector<NormalizedLandmark>.
+typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
+    BeginLoopNormalizedLandmarkCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);
+
+// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
+typedef BeginLoopCalculator<
+    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
+    BeginLoopNormalizedLandmarksVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
+
+// A calculator to process std::vector<NormalizedRect>.
+typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
+    BeginLoopNormalizedRectCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.h

@@ -0,0 +1,157 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator for implementing loops on iterable collections inside a MediaPipe
+// graph.
+//
+// It is designed to be used like:
+//
+// node {
+//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "ElementToBlaConverterSubgraph"
+//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "EndLoopWithOutputCalculator"
+//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+// }
+//
+// BeginLoopCalculator accepts an optional input stream tagged with "TICK"
+// which if non-empty, wakes up the calculator and calls
+// BeginLoopCalculator::Process(). Input streams tagged with "CLONE" are cloned
+// to the corresponding output streams at loop timestamps. This ensures that a
+// MediaPipe graph or sub-graph can run multiple times, once per element in the
+// "ITERABLE" for each pakcet clone of the packets in the "CLONE" input streams.
+template <typename IterableT>
+class BeginLoopCalculator : public CalculatorBase {
+  using ItemT = typename IterableT::value_type;
+
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    // A non-empty packet in the optional "TICK" input stream wakes up the
+    // calculator.
+    if (cc->Inputs().HasTag("TICK")) {
+      cc->Inputs().Tag("TICK").SetAny();
+    }
+
+    // An iterable collection in the input stream.
+    RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
+    cc->Inputs().Tag("ITERABLE").Set<IterableT>();
+
+    // An element from the collection.
+    RET_CHECK(cc->Outputs().HasTag("ITEM"));
+    cc->Outputs().Tag("ITEM").Set<ItemT>();
+
+    RET_CHECK(cc->Outputs().HasTag("BATCH_END"));
+    cc->Outputs()
+        .Tag("BATCH_END")
+        .Set<Timestamp>(
+            // A flush signal to the corresponding EndLoopCalculator for it to
+            // emit the aggregated result with the timestamp contained in this
+            // flush signal packet.
+        );
+
+    // Input streams tagged with "CLONE" are cloned to the corresponding
+    // "CLONE" output streams at loop timestamps.
+    RET_CHECK(cc->Inputs().NumEntries("CLONE") ==
+              cc->Outputs().NumEntries("CLONE"));
+    if (cc->Inputs().NumEntries("CLONE") > 0) {
+      for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
+        cc->Inputs().Get("CLONE", i).SetAny();
+        cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i));
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    Timestamp last_timestamp = loop_internal_timestamp_;
+    if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
+      const IterableT& collection =
+          cc->Inputs().Tag("ITERABLE").template Get<IterableT>();
+      for (const auto& item : collection) {
+        cc->Outputs().Tag("ITEM").AddPacket(
+            MakePacket<ItemT>(item).At(loop_internal_timestamp_));
+        ForwardClonePackets(cc, loop_internal_timestamp_);
+        ++loop_internal_timestamp_;
+      }
+    }
+
+    // The collection was empty and nothing was processed.
+    if (last_timestamp == loop_internal_timestamp_) {
+      // Increment loop_internal_timestamp_ because it is used up now.
+      ++loop_internal_timestamp_;
+      for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) {
+        it->SetNextTimestampBound(loop_internal_timestamp_);
+      }
+    }
+
+    // The for loop processing the input collection already incremented
+    // loop_internal_timestamp_. To emit BATCH_END packet along the last
+    // non-BATCH_END packet, decrement by one.
+    cc->Outputs()
+        .Tag("BATCH_END")
+        .AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
+                       .At(Timestamp(loop_internal_timestamp_ - 1)));
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) {
+    if (cc->Inputs().NumEntries("CLONE") > 0) {
+      for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
+        if (!cc->Inputs().Get("CLONE", i).IsEmpty()) {
+          auto input_packet = cc->Inputs().Get("CLONE", i).Value();
+          cc->Outputs()
+              .Get("CLONE", i)
+              .AddPacket(std::move(input_packet).At(output_timestamp));
+        }
+      }
+    }
+  }
+
+  // Fake timestamps generated per element in collection.
+  Timestamp loop_internal_timestamp_ = Timestamp(0);
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_

mediapipe/calculators/core/clip_detection_vector_size_calculator.cc

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<::mediapipe::Detection>
+    ClipDetectionVectorSizeCalculator;
+REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/clip_vector_size_calculator.cc

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect>
+    ClipNormalizedRectVectorSizeCalculator;
+REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/clip_vector_size_calculator.h

@@ -0,0 +1,137 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
+
+#include <type_traits>
+#include <vector>
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Clips the size of the input vector of type T to a specified max_vec_size.
+// In a graph it will be used as:
+// node {
+//   calculator: "ClipIntVectorSizeCalculator"
+//   input_stream: "input_vector"
+//   output_stream: "output_vector"
+//   options {
+//     [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] {
+//       max_vec_size: 5
+//     }
+//   }
+// }
+template <typename T>
+class ClipVectorSizeCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() == 1);
+
+    if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
+            .max_vec_size() < 1) {
+      return ::mediapipe::InternalError(
+          "max_vec_size should be greater than or equal to 1.");
+    }
+
+    cc->Inputs().Index(0).Set<std::vector<T>>();
+    cc->Outputs().Index(0).Set<std::vector<T>>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
+                        .max_vec_size();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (max_vec_size_ < 1) {
+      return ::mediapipe::InternalError(
+          "max_vec_size should be greater than or equal to 1.");
+    }
+    if (cc->Inputs().Index(0).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    const std::vector<U>& input_vector =
+        cc->Inputs().Index(0).Get<std::vector<U>>();
+    if (max_vec_size_ >= input_vector.size()) {
+      output->insert(output->end(), input_vector.begin(), input_vector.end());
+    } else {
+      for (int i = 0; i < max_vec_size_; ++i) {
+        output->push_back(input_vector[i]);
+      }
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
+    return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
+                                               CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+        cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
+
+    if (input_status.ok()) {
+      std::unique_ptr<std::vector<U>> input_vector =
+          std::move(input_status).ValueOrDie();
+      auto begin_it = input_vector->begin();
+      auto end_it = input_vector->end();
+      if (max_vec_size_ < input_vector->size()) {
+        end_it = input_vector->begin() + max_vec_size_;
+      }
+      output->insert(output->end(), std::make_move_iterator(begin_it),
+                     std::make_move_iterator(end_it));
+    } else {
+      return input_status.status();
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
+                                               CalculatorContext* cc) {
+    return ::mediapipe::InternalError(
+        "Cannot copy or move input vectors and clip their size.");
+  }
+
+ private:
+  int max_vec_size_ = 0;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_

mediapipe/calculators/core/clip_vector_size_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ClipVectorSizeCalculatorOptions {
+  extend CalculatorOptions {
+    optional ClipVectorSizeCalculatorOptions ext = 274674998;
+  }
+
+  // Maximum size of output vector.
+  optional int32 max_vec_size = 1 [default = 1];
+}

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -0,0 +1,179 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<int> TestClipIntVectorSizeCalculator;
+REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator);
+
+void AddInputVector(const std::vector<int>& input, int64 timestamp,
+                    CalculatorRunner* runner) {
+  runner->MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  std::vector<int> input = {};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  std::vector<int> input = {0, 1, 2, 3};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+  EXPECT_EQ(2, output.size());
+  std::vector<int> expected_vector = {0, 1};
+  EXPECT_EQ(expected_vector, output);
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  {
+    std::vector<int> input = {0, 1, 2, 3};
+    AddInputVector(input, /*timestamp=*/1, &runner);
+  }
+  {
+    std::vector<int> input = {2, 3, 4, 5};
+    AddInputVector(input, /*timestamp=*/2, &runner);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(2, outputs.size());
+  {
+    EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+    const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+    EXPECT_EQ(3, output.size());
+    std::vector<int> expected_vector = {0, 1, 2};
+    EXPECT_EQ(expected_vector, output);
+  }
+  {
+    EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
+    const std::vector<int>& output = outputs[1].Get<std::vector<int>>();
+    EXPECT_EQ(3, output.size());
+    std::vector<int> expected_vector = {2, 3, 4};
+    EXPECT_EQ(expected_vector, output);
+  }
+}
+
+typedef ClipVectorSizeCalculator<std::unique_ptr<int>>
+    TestClipUniqueIntPtrVectorSizeCalculator;
+REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator);
+
+TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "input_vector"
+        node {
+          calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
+          input_stream: "input_vector"
+          output_stream: "output_vector"
+          options {
+            [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
+          }
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("output_vector", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2, 3, 4, 5}
+  auto input_vector = absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(3, result.size());
+  for (int i = 0; i < 3; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/concatenate_detection_vector_calculator.cc

@@ -0,0 +1,33 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "ConcatenateDetectionVectorCalculator"
+//   input_stream: "detection_vector_1"
+//   input_stream: "detection_vector_2"
+//   output_stream: "concatenated_detection_vector"
+// }
+typedef ConcatenateVectorCalculator<::mediapipe::Detection>
+    ConcatenateDetectionVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateDetectionVectorCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -0,0 +1,59 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "tensorflow/lite/interpreter.h"
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "ConcatenateFloatVectorCalculator"
+//   input_stream: "float_vector_1"
+//   input_stream: "float_vector_2"
+//   output_stream: "concatenated_float_vector"
+// }
+typedef ConcatenateVectorCalculator<float> ConcatenateFloatVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateFloatVectorCalculator);
+
+// Example config:
+// node {
+//   calculator: "ConcatenateTfLiteTensorVectorCalculator"
+//   input_stream: "tflitetensor_vector_1"
+//   input_stream: "tflitetensor_vector_2"
+//   output_stream: "concatenated_tflitetensor_vector"
+// }
+typedef ConcatenateVectorCalculator<TfLiteTensor>
+    ConcatenateTfLiteTensorVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
+
+typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
+    ConcatenateLandmarkVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
+    ConcatenateGlBufferVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
+#endif
+
+}  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.h

@@ -0,0 +1,121 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
+
+#include <type_traits>
+#include <vector>
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Concatenates several std::vector<T> following stream index order. This class
+// assumes that every input stream contains the vector<T> type. To use this
+// class for a particular type T, regisiter a calculator using
+// ConcatenateVectorCalculator<T>.
+template <typename T>
+class ConcatenateVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() != 0);
+    RET_CHECK(cc->Outputs().NumEntries() == 1);
+
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      cc->Inputs().Index(i).Set<std::vector<T>>();
+    }
+
+    cc->Outputs().Index(0).Set<std::vector<T>>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    only_emit_if_all_present_ =
+        cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>()
+            .only_emit_if_all_present();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (only_emit_if_all_present_) {
+      for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+        if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
+      }
+    }
+
+    return ConcatenateVectors<T>(std::is_copy_constructible<T>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConcatenateVectors(std::true_type,
+                                         CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      if (cc->Inputs().Index(i).IsEmpty()) continue;
+      const std::vector<U>& input = cc->Inputs().Index(i).Get<std::vector<U>>();
+      output->insert(output->end(), input.begin(), input.end());
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConcatenateVectors(std::false_type,
+                                         CalculatorContext* cc) {
+    return ConsumeAndConcatenateVectors<T>(std::is_move_constructible<U>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
+                                                   CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      if (cc->Inputs().Index(i).IsEmpty()) continue;
+      ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+          cc->Inputs().Index(i).Value().Consume<std::vector<U>>();
+      if (input_status.ok()) {
+        std::unique_ptr<std::vector<U>> input_vector =
+            std::move(input_status).ValueOrDie();
+        output->insert(output->end(),
+                       std::make_move_iterator(input_vector->begin()),
+                       std::make_move_iterator(input_vector->end()));
+      } else {
+        return input_status.status();
+      }
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
+                                                   CalculatorContext* cc) {
+    return ::mediapipe::InternalError(
+        "Cannot copy or move input vectors to concatenate them");
+  }
+
+ private:
+  bool only_emit_if_all_present_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_

mediapipe/calculators/core/concatenate_vector_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ConcatenateVectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional ConcatenateVectorCalculatorOptions ext = 259397839;
+  }
+
+  // If true, the calculator will only emit a packet at the given timestamp if
+  // all input streams have a non-empty packet (AND operation on streams).
+  optional bool only_emit_if_all_present = 1 [default = false];
+}

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -0,0 +1,401 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/concatenate_vector_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
+REGISTER_CALCULATOR(TestConcatenateIntVectorCalculator);
+
+void AddInputVectors(const std::vector<std::vector<int>>& inputs,
+                     int64 timestamp, CalculatorRunner* runner) {
+  for (int i = 0; i < inputs.size(); ++i) {
+    runner->MutableInputs()->Index(i).packets.push_back(
+        MakePacket<std::vector<int>>(inputs[i]).At(Timestamp(timestamp)));
+  }
+}
+
+TEST(TestConcatenateIntVectorCalculatorTest, EmptyVectorInputs) {
+  CalculatorRunner runner("TestConcatenateIntVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<int>> inputs = {{}, {}, {}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+}
+
+TEST(TestConcatenateIntVectorCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("TestConcatenateIntVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<int>> inputs = {{1, 2, 3}, {4}, {5, 6}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
+}
+
+TEST(TestConcatenateIntVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
+  CalculatorRunner runner("TestConcatenateIntVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+  {
+    std::vector<std::vector<int>> inputs = {{1, 2, 3}, {4}, {5, 6}};
+    AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  }
+  {
+    std::vector<std::vector<int>> inputs = {{0, 2}, {1}, {3, 5}};
+    AddInputVectors(inputs, /*timestamp=*/2, &runner);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(2, outputs.size());
+  {
+    EXPECT_EQ(6, outputs[0].Get<std::vector<int>>().size());
+    EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+    std::vector<int> expected_vector = {1, 2, 3, 4, 5, 6};
+    EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
+  }
+  {
+    EXPECT_EQ(5, outputs[1].Get<std::vector<int>>().size());
+    EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
+    std::vector<int> expected_vector = {0, 2, 1, 3, 5};
+    EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<int>>());
+  }
+}
+
+TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
+  CalculatorRunner runner("TestConcatenateIntVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<int>> inputs = {{1, 2, 3}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<int> expected_vector = {1, 2, 3};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<int>>());
+}
+
+TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
+  CalculatorRunner runner("TestConcatenateIntVectorCalculator",
+                          /*options_string=*/
+                          "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
+                          "{only_emit_if_all_present: true}",
+                          /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<int>> inputs = {{1, 2, 3}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(0, outputs.size());
+}
+
+void AddInputVectors(const std::vector<std::vector<float>>& inputs,
+                     int64 timestamp, CalculatorRunner* runner) {
+  for (int i = 0; i < inputs.size(); ++i) {
+    runner->MutableInputs()->Index(i).packets.push_back(
+        MakePacket<std::vector<float>>(inputs[i]).At(Timestamp(timestamp)));
+  }
+}
+
+TEST(ConcatenateFloatVectorCalculatorTest, EmptyVectorInputs) {
+  CalculatorRunner runner("ConcatenateFloatVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<float>> inputs = {{}, {}, {}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_TRUE(outputs[0].Get<std::vector<float>>().empty());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+}
+
+TEST(ConcatenateFloatVectorCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("ConcatenateFloatVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<float>> inputs = {
+      {1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
+}
+
+TEST(ConcatenateFloatVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
+  CalculatorRunner runner("ConcatenateFloatVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+  {
+    std::vector<std::vector<float>> inputs = {
+        {1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}};
+    AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  }
+  {
+    std::vector<std::vector<float>> inputs = {
+        {0.0f, 2.0f}, {1.0f}, {3.0f, 5.0f}};
+    AddInputVectors(inputs, /*timestamp=*/2, &runner);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(2, outputs.size());
+  {
+    EXPECT_EQ(6, outputs[0].Get<std::vector<float>>().size());
+    EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+    std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f};
+    EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
+  }
+  {
+    EXPECT_EQ(5, outputs[1].Get<std::vector<float>>().size());
+    EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
+    std::vector<float> expected_vector = {0.0f, 2.0f, 1.0f, 3.0f, 5.0f};
+    EXPECT_EQ(expected_vector, outputs[1].Get<std::vector<float>>());
+  }
+}
+
+TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamStillOutput) {
+  CalculatorRunner runner("ConcatenateFloatVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<float> expected_vector = {1.0f, 2.0f, 3.0f};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<float>>());
+}
+
+TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
+  CalculatorRunner runner("ConcatenateFloatVectorCalculator",
+                          /*options_string=*/
+                          "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: "
+                          "{only_emit_if_all_present: true}",
+                          /*num_inputs=*/2,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(0, outputs.size());
+}
+
+typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
+    TestConcatenateUniqueIntPtrCalculator;
+REGISTER_CALCULATOR(TestConcatenateUniqueIntPtrCalculator);
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        input_stream: "in_3"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          input_stream: "in_3"
+          output_stream: "out"
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+  // input2: {3}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_2 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(1);
+  input_2->at(0) = absl::make_unique<int>(3);
+  // input3: {4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_3 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(2);
+  input_3->at(0) = absl::make_unique<int>(4);
+  input_3->at(1) = absl::make_unique<int>(5);
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_2", Adopt(input_2.release()).At(Timestamp(1))));
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_3", Adopt(input_3.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(6, result.size());
+  for (int i = 0; i < 6; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          output_stream: "out"
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(3, result.size());
+  for (int i = 0; i < 3; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          output_stream: "out"
+          options {
+            [mediapipe.ConcatenateVectorCalculatorOptions.ext] {
+              only_emit_if_all_present: true
+            }
+          }
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(0, outputs.size());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator.cc

@@ -0,0 +1,90 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cfloat>
+
+#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/status.h"
+
+// Dequantizes a byte array to a vector of floats.
+//
+// Example config:
+//   node {
+//     calculator: "DequantizeByteArrayCalculator"
+//     input_stream: "ENCODED:encoded"
+//     output_stream: "FLOAT_VECTOR:float_vector"
+//     options {
+//       [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+//         max_quantized_value: 2
+//         min_quantized_value: -2
+//       }
+//     }
+//   }
+namespace mediapipe {
+
+class DequantizeByteArrayCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag("ENCODED").Set<std::string>();
+    cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    const auto options =
+        cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
+    if (!options.has_max_quantized_value() ||
+        !options.has_min_quantized_value()) {
+      return ::mediapipe::InvalidArgumentError(
+          "Both max_quantized_value and min_quantized_value must be provided "
+          "in DequantizeByteArrayCalculatorOptions.");
+    }
+    float max_quantized_value = options.max_quantized_value();
+    float min_quantized_value = options.min_quantized_value();
+    if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
+      return ::mediapipe::InvalidArgumentError(
+          "max_quantized_value must be greater than min_quantized_value.");
+    }
+    float range = max_quantized_value - min_quantized_value;
+    scalar_ = range / 255.0;
+    bias_ = (range / 512.0) + min_quantized_value;
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    const std::string& encoded =
+        cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
+    std::vector<float> float_vector;
+    float_vector.reserve(encoded.length());
+    for (int i = 0; i < encoded.length(); ++i) {
+      float_vector.push_back(
+          static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
+    }
+    cc->Outputs()
+        .Tag("FLOAT_VECTOR")
+        .AddPacket(MakePacket<std::vector<float>>(float_vector)
+                       .At(cc->InputTimestamp()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float scalar_;
+  float bias_;
+};
+
+REGISTER_CALCULATOR(DequantizeByteArrayCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message DequantizeByteArrayCalculatorOptions {
+  extend CalculatorOptions {
+    optional DequantizeByteArrayCalculatorOptions ext = 272316343;
+  }
+
+  optional float max_quantized_value = 1;
+  optional float min_quantized_value = 2;
+}

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -0,0 +1,137 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "Both max_quantized_value and min_quantized_value must be provided"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: -2
+            min_quantized_value: 2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 1
+            min_quantized_value: 1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 2
+            min_quantized_value: -2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(
+          std::string(reinterpret_cast<char const*>(input), 4))
+          .At(Timestamp(0)));
+  auto status = runner.Run();
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs =
+      runner.Outputs().Tag("FLOAT_VECTOR").packets;
+  EXPECT_EQ(1, outputs.size());
+  const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
+  ASSERT_FALSE(result.empty());
+  EXPECT_EQ(4, result.size());
+  EXPECT_NEAR(0, result[0], 0.01);
+  EXPECT_NEAR(2, result[1], 0.01);
+  EXPECT_NEAR(-2, result[2], 0.01);
+  EXPECT_NEAR(-1.976, result[3], 0.01);
+
+  EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.cc

@@ -0,0 +1,45 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/end_loop_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/util/render_data.pb.h"
+
+namespace mediapipe {
+
+typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
+    EndLoopNormalizedRectCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
+
+typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
+    EndLoopNormalizedLandmarkCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);
+
+typedef EndLoopCalculator<
+    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
+    EndLoopNormalizedLandmarksVectorCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
+
+typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
+REGISTER_CALCULATOR(EndLoopBooleanCalculator);
+
+typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
+    EndLoopRenderDataCalculator;
+REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
+
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator for completing the processing of loops on iterable collections
+// inside a MediaPipe graph. The EndLoopCalculator collects all input packets
+// from ITEM input_stream into a collection and upon receiving the flush signal
+// from the "BATCH_END" tagged input stream, it emits the aggregated results
+// at the original timestamp contained in the "BATCH_END" input stream.
+//
+// It is designed to be used like:
+//
+// node {
+//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "ElementToBlaConverterSubgraph"
+//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "EndLoopWithOutputCalculator"
+//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+// }
+template <typename IterableT>
+class EndLoopCalculator : public CalculatorBase {
+  using ItemT = typename IterableT::value_type;
+
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
+        << "Missing BATCH_END tagged input_stream.";
+    cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
+
+    RET_CHECK(cc->Inputs().HasTag("ITEM"));
+    cc->Inputs().Tag("ITEM").Set<ItemT>();
+
+    RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
+    cc->Outputs().Tag("ITERABLE").Set<IterableT>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
+      if (!input_stream_collection_) {
+        input_stream_collection_.reset(new IterableT);
+      }
+      input_stream_collection_->push_back(
+          cc->Inputs().Tag("ITEM").template Get<ItemT>());
+    }
+
+    if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) {  // flush signal
+      Timestamp loop_control_ts =
+          cc->Inputs().Tag("BATCH_END").template Get<Timestamp>();
+      if (input_stream_collection_) {
+        cc->Outputs()
+            .Tag("ITERABLE")
+            .Add(input_stream_collection_.release(), loop_control_ts);
+      } else {
+        // Since there is no collection, inform downstream calculators to not
+        // expect any packet by updating the timestamp bounds.
+        cc->Outputs()
+            .Tag("ITERABLE")
+            .SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::unique_ptr<IterableT> input_stream_collection_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_

mediapipe/calculators/core/flow_limiter_calculator.cc

@@ -23,34 +23,34 @@
 
 namespace mediapipe {
 
-// RealTimeFlowLimiterCalculator is used to limit the number of pipelined
+// FlowLimiterCalculator is used to limit the number of pipelined processing
-// processing operations in a section of the graph.
+// operations in a section of the graph.
 //
 // Typical topology:
 //
-// in ->-[RTFLC]-[foo]-...-[bar]-+->- out
+// in ->-[FLC]-[foo]-...-[bar]-+->- out
-//          ^____________________|
+//         ^_____________________|
-//       FINISHED
+//      FINISHED
 //
 // By connecting the output of the graph section to this calculator's FINISHED
-// input with a backwards edge, this allows RTFLC to keep track of how many
+// input with a backwards edge, this allows FLC to keep track of how many
 // timestamps are currently being processed.
 //
 // The limit defaults to 1, and can be overridden with the MAX_IN_FLIGHT side
 // packet.
 //
 // As long as the number of timestamps being processed ("in flight") is below
-// the limit, RTFLC allows input to pass through. When the limit is reached,
+// the limit, FLC allows input to pass through. When the limit is reached,
-// RTFLC starts dropping input packets, keeping only the most recent. When the
+// FLC starts dropping input packets, keeping only the most recent. When the
 // processing count decreases again, as signaled by the receipt of a packet on
-// FINISHED, RTFLC allows packets to flow again, releasing the most recently
+// FINISHED, FLC allows packets to flow again, releasing the most recently
 // queued packet, if any.
 //
 // If there are multiple input streams, packet dropping is synchronized.
 //
-// IMPORTANT: for each timestamp where RTFLC forwards a packet (or a set of
+// IMPORTANT: for each timestamp where FLC forwards a packet (or a set of
 // packets, if using multiple data streams), a packet must eventually arrive on
-// the FINISHED stream. Dropping packets in the section between RTFLC and
+// the FINISHED stream. Dropping packets in the section between FLC and
 // FINISHED will make the in-flight count incorrect.
 //
 // TODO: Remove this comment when graph-level ISH has been removed.
@@ -61,7 +61,7 @@ namespace mediapipe {
 //
 // Example config:
 // node {
-//   calculator: "RealTimeFlowLimiterCalculator"
+//   calculator: "FlowLimiterCalculator"
 //   input_stream: "raw_frames"
 //   input_stream: "FINISHED:finished"
 //   input_stream_info: {
@@ -73,7 +73,7 @@ namespace mediapipe {
 //   }
 //   output_stream: "gated_frames"
 // }
-class RealTimeFlowLimiterCalculator : public CalculatorBase {
+class FlowLimiterCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
     int num_data_streams = cc->Inputs().NumEntries("");
@@ -194,6 +194,6 @@ class RealTimeFlowLimiterCalculator : public CalculatorBase {
   Timestamp allow_ctr_ts_;
   std::vector<Timestamp> data_stream_bound_ts_;
 };
-REGISTER_CALCULATOR(RealTimeFlowLimiterCalculator);
+REGISTER_CALCULATOR(FlowLimiterCalculator);
 
 }  // namespace mediapipe

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -71,7 +71,7 @@ constexpr int kNumImageFrames = 5;
 constexpr int kNumFinished = 3;
 CalculatorGraphConfig::Node GetDefaultNode() {
   return ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
-    calculator: "RealTimeFlowLimiterCalculator"
+    calculator: "FlowLimiterCalculator"
     input_stream: "raw_frames"
     input_stream: "FINISHED:finished"
     input_stream_info: { tag_index: "FINISHED" back_edge: true }
@@ -79,9 +79,9 @@ CalculatorGraphConfig::Node GetDefaultNode() {
   )");
 }
 
-// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs
+// Simple test to make sure that the FlowLimiterCalculator outputs just one
-// just one packet when MAX_IN_FLIGHT is 1.
+// packet when MAX_IN_FLIGHT is 1.
-TEST(RealTimeFlowLimiterCalculator, OneOutputTest) {
+TEST(FlowLimiterCalculator, OneOutputTest) {
   // Setup the calculator runner and add only ImageFrame packets.
   CalculatorRunner runner(GetDefaultNode());
   for (int i = 0; i < kNumImageFrames; ++i) {
@@ -91,16 +91,16 @@ TEST(RealTimeFlowLimiterCalculator, OneOutputTest) {
   }
 
   // Run the calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& frame_output_packets =
       runner.Outputs().Index(0).packets;
 
   EXPECT_EQ(frame_output_packets.size(), 1);
 }
 
-// Simple test to make sure that the RealTimeFlowLimiterCalculator waits for all
+// Simple test to make sure that the FlowLimiterCalculator waits for all
 // input streams to have at least one packet available before publishing.
-TEST(RealTimeFlowLimiterCalculator, BasicTest) {
+TEST(FlowLimiterCalculator, BasicTest) {
   // Setup the calculator runner and add both ImageFrame and finish packets.
   CalculatorRunner runner(GetDefaultNode());
   for (int i = 0; i < kNumImageFrames; ++i) {
@@ -117,7 +117,7 @@ TEST(RealTimeFlowLimiterCalculator, BasicTest) {
   }
 
   // Run the calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& frame_output_packets =
       runner.Outputs().Index(0).packets;
 
@@ -171,13 +171,11 @@ class CloseCallbackCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(CloseCallbackCalculator);
 
-// Tests demostrating an RealTimeFlowLimiterCalculator operating in a cyclic
+// Tests demostrating an FlowLimiterCalculator operating in a cyclic graph.
-// graph.
 // TODO: clean up these tests.
-class RealTimeFlowLimiterCalculatorTest : public testing::Test {
+class FlowLimiterCalculatorTest : public testing::Test {
  public:
-  RealTimeFlowLimiterCalculatorTest()
+  FlowLimiterCalculatorTest() : enter_semaphore_(0), exit_semaphore_(0) {}
-      : enter_semaphore_(0), exit_semaphore_(0) {}
 
   void SetUp() override {
     graph_config_ = InflightGraphConfig();
@@ -200,7 +198,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
       close_count_++;
       return ::mediapipe::OkStatus();
     };
-    MEDIAPIPE_ASSERT_OK(graph_.Initialize(
+    MP_ASSERT_OK(graph_.Initialize(
         graph_config_, {
                            {"max_in_flight", MakePacket<int>(max_in_flight)},
                            {"callback_0", Adopt(new auto(semaphore_0_func))},
@@ -211,11 +209,11 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
 
   // Adds a packet to a graph input stream.
   void AddPacket(const std::string& input_name, int value) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(value).At(Timestamp(value))));
   }
 
-  // A calculator graph starting with an RealTimeFlowLimiterCalculator and
+  // A calculator graph starting with an FlowLimiterCalculator and
   // ending with a InFlightFinishCalculator.
   // Back-edge "finished" limits processing to one frame in-flight.
   // The two LambdaCalculators are used to keep certain packet sets in flight.
@@ -224,7 +222,7 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
       input_stream: 'in_1'
       input_stream: 'in_2'
       node {
-        calculator: 'RealTimeFlowLimiterCalculator'
+        calculator: 'FlowLimiterCalculator'
         input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
         input_stream: 'in_1'
         input_stream: 'in_2'
@@ -270,19 +268,19 @@ class RealTimeFlowLimiterCalculatorTest : public testing::Test {
   int close_count_ = 0;
 };
 
-// A test demonstrating an RealTimeFlowLimiterCalculator operating in a cyclic
+// A test demonstrating an FlowLimiterCalculator operating in a cyclic
 // graph. This test shows that:
 //
 // (1) Timestamps are passed through unaltered.
 // (2) All output streams including the back_edge stream are closed when
 //     the first input stream is closed.
 //
-TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
+TEST_F(FlowLimiterCalculatorTest, BackEdgeCloses) {
   InitializeGraph(1);
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   auto send_packet = [this](const std::string& input_name, int64 n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int64>(n).At(Timestamp(n))));
   };
 
@@ -290,14 +288,14 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
     send_packet("in_1", i * 10);
     // This next input should be dropped.
     send_packet("in_1", i * 10 + 5);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
     send_packet("in_2", i * 10);
     exit_semaphore_.Release(1);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
   }
-  MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_1"));
+  MP_EXPECT_OK(graph_.CloseInputStream("in_1"));
-  MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_2"));
+  MP_EXPECT_OK(graph_.CloseInputStream("in_2"));
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
 
   // All output streams are closed and all output packets are delivered,
   // with stream "in_1" and stream "in_2" closed.
@@ -321,19 +319,19 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) {
 
 // A test demonstrating that all output streams are closed when all
 // input streams are closed after the last input packet has been processed.
-TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
+TEST_F(FlowLimiterCalculatorTest, AllStreamsClose) {
   InitializeGraph(1);
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   exit_semaphore_.Release(10);
   for (int i = 0; i < 10; i++) {
     AddPacket("in_1", i);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
     AddPacket("in_2", i);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
   }
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
 
   EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
   EXPECT_EQ(TimestampValues(out_1_packets_),
@@ -341,7 +339,7 @@ TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) {
   EXPECT_EQ(1, close_count_);
 }
 
-TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
+TEST(FlowLimiterCalculator, TwoStreams) {
   std::vector<Packet> a_passed;
   std::vector<Packet> b_passed;
   CalculatorGraphConfig graph_config_ =
@@ -351,7 +349,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
         input_stream: 'finished'
         node {
           name: 'input_dropper'
-          calculator: 'RealTimeFlowLimiterCalculator'
+          calculator: 'FlowLimiterCalculator'
           input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
           input_stream: 'in_a'
           input_stream: 'in_b'
@@ -373,7 +371,7 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
   };
 
   CalculatorGraph graph_;
-  MEDIAPIPE_EXPECT_OK(graph_.Initialize(
+  MP_EXPECT_OK(graph_.Initialize(
       graph_config_,
       {
           {"max_in_flight", MakePacket<int>(1)},
@@ -381,66 +379,66 @@ TEST(RealTimeFlowLimiterCalculator, TwoStreams) {
            MakePacket<std::function<void(const Packet&)>>(allow_cb)},
       }));
 
-  MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
+  MP_EXPECT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   send_packet("in_a", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(allow, false);
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{}));
 
   send_packet("in_a", 2);
   send_packet("in_b", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, false);
 
   send_packet("finished", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, true);
 
   send_packet("in_b", 2);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, true);
 
   send_packet("in_b", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("in_b", 4);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("in_a", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("finished", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, true);
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
-TEST(RealTimeFlowLimiterCalculator, CanConsume) {
+TEST(FlowLimiterCalculator, CanConsume) {
   std::vector<Packet> in_sampled_packets_;
   CalculatorGraphConfig graph_config_ =
       ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
@@ -448,7 +446,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
         input_stream: 'finished'
         node {
           name: 'input_dropper'
-          calculator: 'RealTimeFlowLimiterCalculator'
+          calculator: 'FlowLimiterCalculator'
           input_side_packet: 'MAX_IN_FLIGHT:max_in_flight'
           input_stream: 'in'
           input_stream: 'FINISHED:finished'
@@ -467,7 +465,7 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
   };
 
   CalculatorGraph graph_;
-  MEDIAPIPE_EXPECT_OK(graph_.Initialize(
+  MP_EXPECT_OK(graph_.Initialize(
       graph_config_,
       {
           {"max_in_flight", MakePacket<int>(1)},
@@ -475,21 +473,21 @@ TEST(RealTimeFlowLimiterCalculator, CanConsume) {
            MakePacket<std::function<void(const Packet&)>>(allow_cb)},
       }));
 
-  MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
+  MP_EXPECT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   send_packet("in", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(allow, false);
   EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64>{1}));
 
-  MEDIAPIPE_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
+  MP_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
 }  // anonymous namespace

mediapipe/calculators/core/gate_calculator.cc

@@ -0,0 +1,163 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/gate_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/header_util.h"
+
+namespace mediapipe {
+
+namespace {
+enum GateState {
+  GATE_UNINITIALIZED,
+  GATE_ALLOW,
+  GATE_DISALLOW,
+};
+
+std::string ToString(GateState state) {
+  switch (state) {
+    case GATE_UNINITIALIZED:
+      return "UNINITIALIZED";
+    case GATE_ALLOW:
+      return "ALLOW";
+    case GATE_DISALLOW:
+      return "DISALLOW";
+  }
+  DLOG(FATAL) << "Unknown GateState";
+  return "UNKNOWN";
+}
+}  // namespace
+
+// Controls whether or not the input packets are passed further along the graph.
+// Takes multiple data input streams and either an ALLOW or a DISALLOW control
+// input stream. It outputs an output stream for each input stream that is not
+// ALLOW or DISALLOW as well as an optional STATE_CHANGE stream which downstream
+// calculators can use to respond to state-change events.
+//
+// If the current ALLOW packet is set to true, the input packets are passed to
+// their corresponding output stream unchanged. If the ALLOW packet is set to
+// false, the current input packet is NOT passed to the output stream. If using
+// DISALLOW, the behavior is opposite of ALLOW.
+//
+// By default, an empty packet in the ALLOW or DISALLOW input stream indicates
+// disallowing the corresponding packets in other input streams. The behavior
+// can be inverted with a calculator option.
+//
+// Intended to be used with the default input stream handler, which synchronizes
+// all data input streams with the ALLOW/DISALLOW control input stream.
+//
+// Example config:
+// node {
+//   calculator: "GateCalculator"
+//   input_stream: "input_stream0"
+//   input_stream: "input_stream1"
+//   input_stream: "input_streamN"
+//   input_stream: "ALLOW:allow" or "DISALLOW:disallow"
+//   output_stream: "STATE_CHANGE:state_change"
+//   output_stream: "output_stream0"
+//   output_stream: "output_stream1"
+//   output_stream: "output_streamN"
+// }
+class GateCalculator : public CalculatorBase {
+ public:
+  GateCalculator() {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    // Assume that input streams do not have a tag and that gating signal is
+    // tagged either ALLOW or DISALLOW.
+    RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
+    const int num_data_streams = cc->Inputs().NumEntries("");
+    RET_CHECK_GE(num_data_streams, 1);
+    RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams)
+        << "Number of data output streams must match with data input streams.";
+
+    for (int i = 0; i < num_data_streams; ++i) {
+      cc->Inputs().Get("", i).SetAny();
+      cc->Outputs().Get("", i).SetSameAs(&cc->Inputs().Get("", i));
+    }
+    if (cc->Inputs().HasTag("ALLOW")) {
+      cc->Inputs().Tag("ALLOW").Set<bool>();
+    } else {
+      cc->Inputs().Tag("DISALLOW").Set<bool>();
+    }
+
+    if (cc->Outputs().HasTag("STATE_CHANGE")) {
+      cc->Outputs().Tag("STATE_CHANGE").Set<bool>();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(TimestampDiff(0));
+    num_data_streams_ = cc->Inputs().NumEntries("");
+    last_gate_state_ = GATE_UNINITIALIZED;
+    RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
+
+    const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
+    empty_packets_as_allow_ = options.empty_packets_as_allow();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    bool allow = empty_packets_as_allow_;
+    if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) {
+      allow = cc->Inputs().Tag("ALLOW").Get<bool>();
+    }
+    if (cc->Inputs().HasTag("DISALLOW") &&
+        !cc->Inputs().Tag("DISALLOW").IsEmpty()) {
+      allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
+    }
+
+    const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
+
+    if (cc->Outputs().HasTag("STATE_CHANGE")) {
+      if (last_gate_state_ != GATE_UNINITIALIZED &&
+          last_gate_state_ != new_gate_state) {
+        VLOG(2) << "State transition in " << cc->NodeName() << " @ "
+                << cc->InputTimestamp().Value() << " from "
+                << ToString(last_gate_state_) << " to "
+                << ToString(new_gate_state);
+        cc->Outputs()
+            .Tag("STATE_CHANGE")
+            .AddPacket(MakePacket<bool>(allow).At(cc->InputTimestamp()));
+      }
+    }
+    last_gate_state_ = new_gate_state;
+
+    if (!allow) {
+      return ::mediapipe::OkStatus();
+    }
+
+    // Process data streams.
+    for (int i = 0; i < num_data_streams_; ++i) {
+      if (!cc->Inputs().Get("", i).IsEmpty()) {
+        cc->Outputs().Get("", i).AddPacket(cc->Inputs().Get("", i).Value());
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  GateState last_gate_state_ = GATE_UNINITIALIZED;
+  int num_data_streams_;
+  bool empty_packets_as_allow_;
+};
+REGISTER_CALCULATOR(GateCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/gate_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message GateCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional GateCalculatorOptions ext = 261754847;
+  }
+
+  // By default an empty packet in the ALLOW or DISALLOW input stream indicates
+  // disallowing the corresponding packets in the data input streams. Setting
+  // this option to true inverts that, allowing the data packets to go through.
+  optional bool empty_packets_as_allow = 1;
+}

mediapipe/calculators/core/gate_calculator_test.cc

@@ -0,0 +1,190 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+class GateCalculatorTest : public ::testing::Test {
+ protected:
+  void RunTimeStep(int64 timestamp, const std::string& control_tag,
+                   bool control) {
+    runner_->MutableInputs()->Get("", 0).packets.push_back(
+        MakePacket<bool>(true).At(Timestamp(timestamp)));
+    runner_->MutableInputs()
+        ->Tag(control_tag)
+        .packets.push_back(MakePacket<bool>(control).At(Timestamp(timestamp)));
+
+    MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
+  }
+
+  void SetRunner(const std::string& proto) {
+    runner_ = absl::make_unique<CalculatorRunner>(
+        ParseTextProtoOrDie<CalculatorGraphConfig::Node>(proto));
+  }
+
+  CalculatorRunner* runner() { return runner_.get(); }
+
+ private:
+  std::unique_ptr<CalculatorRunner> runner_;
+};
+
+TEST_F(GateCalculatorTest, Allow) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "ALLOW:gating_stream"
+        output_stream: "test_output"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "ALLOW", true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, "ALLOW", false);
+  constexpr int64 kTimestampValue2 = 44;
+  RunTimeStep(kTimestampValue2, "ALLOW", true);
+  constexpr int64 kTimestampValue3 = 45;
+  RunTimeStep(kTimestampValue3, "ALLOW", false);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(2, output.size());
+  EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value());
+  EXPECT_EQ(kTimestampValue2, output[1].Timestamp().Value());
+  EXPECT_EQ(true, output[0].Get<bool>());
+  EXPECT_EQ(true, output[1].Get<bool>());
+}
+
+TEST_F(GateCalculatorTest, Disallow) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "DISALLOW:gating_stream"
+        output_stream: "test_output"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "DISALLOW", true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, "DISALLOW", false);
+  constexpr int64 kTimestampValue2 = 44;
+  RunTimeStep(kTimestampValue2, "DISALLOW", true);
+  constexpr int64 kTimestampValue3 = 45;
+  RunTimeStep(kTimestampValue3, "DISALLOW", false);
+
+  const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
+  ASSERT_EQ(2, output.size());
+  EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
+  EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
+  EXPECT_EQ(true, output[0].Get<bool>());
+  EXPECT_EQ(true, output[1].Get<bool>());
+}
+
+TEST_F(GateCalculatorTest, AllowWithStateChange) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "ALLOW:gating_stream"
+        output_stream: "test_output"
+        output_stream: "STATE_CHANGE:state_changed"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "ALLOW", false);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, "ALLOW", true);
+  constexpr int64 kTimestampValue2 = 44;
+  RunTimeStep(kTimestampValue2, "ALLOW", true);
+  constexpr int64 kTimestampValue3 = 45;
+  RunTimeStep(kTimestampValue3, "ALLOW", false);
+
+  const std::vector<Packet>& output =
+      runner()->Outputs().Get("STATE_CHANGE", 0).packets;
+  ASSERT_EQ(2, output.size());
+  EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
+  EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
+  EXPECT_EQ(true, output[0].Get<bool>());   // Allow.
+  EXPECT_EQ(false, output[1].Get<bool>());  // Disallow.
+}
+
+TEST_F(GateCalculatorTest, DisallowWithStateChange) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "DISALLOW:gating_stream"
+        output_stream: "test_output"
+        output_stream: "STATE_CHANGE:state_changed"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "DISALLOW", true);
+  constexpr int64 kTimestampValue1 = 43;
+  RunTimeStep(kTimestampValue1, "DISALLOW", false);
+  constexpr int64 kTimestampValue2 = 44;
+  RunTimeStep(kTimestampValue2, "DISALLOW", false);
+  constexpr int64 kTimestampValue3 = 45;
+  RunTimeStep(kTimestampValue3, "DISALLOW", true);
+
+  const std::vector<Packet>& output =
+      runner()->Outputs().Get("STATE_CHANGE", 0).packets;
+  ASSERT_EQ(2, output.size());
+  EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
+  EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
+  EXPECT_EQ(true, output[0].Get<bool>());   // Allow.
+  EXPECT_EQ(false, output[1].Get<bool>());  // Disallow.
+}
+
+// Must not detect disallow value for first timestamp as a state change.
+TEST_F(GateCalculatorTest, DisallowInitialNoStateTransition) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "DISALLOW:gating_stream"
+        output_stream: "test_output"
+        output_stream: "STATE_CHANGE:state_changed"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "DISALLOW", false);
+
+  const std::vector<Packet>& output =
+      runner()->Outputs().Get("STATE_CHANGE", 0).packets;
+  ASSERT_EQ(0, output.size());
+}
+
+// Must not detect allow value for first timestamp as a state change.
+TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
+  SetRunner(R"(
+        calculator: "GateCalculator"
+        input_stream: "test_input"
+        input_stream: "ALLOW:gating_stream"
+        output_stream: "test_output"
+        output_stream: "STATE_CHANGE:state_changed"
+  )");
+
+  constexpr int64 kTimestampValue0 = 42;
+  RunTimeStep(kTimestampValue0, "ALLOW", true);
+
+  const std::vector<Packet>& output =
+      runner()->Outputs().Get("STATE_CHANGE", 0).packets;
+  ASSERT_EQ(0, output.size());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/immediate_mux_calculator_test.cc

@@ -146,7 +146,7 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
     ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"(
         input_stream: "input_packets_0"
         node {
-          calculator: 'RealTimeFlowLimiterCalculator'
+          calculator: 'FlowLimiterCalculator'
           input_stream_handler {
             input_stream_handler: 'ImmediateInputStreamHandler'
           }
@@ -217,23 +217,23 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
 
     // Start running the graph.
     CalculatorGraph graph;
-    MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config_));
+    MP_ASSERT_OK(graph.Initialize(graph_config_));
-    MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+    MP_ASSERT_OK(graph.StartRun({}));
 
     // Send each packet to the graph in the specified order.
     for (int t = 0; t < input_sets.size(); t++) {
       const std::vector<Packet>& input_set = input_sets[t];
-      MEDIAPIPE_EXPECT_OK(graph.WaitUntilIdle());
+      MP_EXPECT_OK(graph.WaitUntilIdle());
       for (int i = 0; i < input_set.size(); i++) {
         const Packet& packet = input_set[i];
         if (!IsNone(packet)) {
-          MEDIAPIPE_EXPECT_OK(graph.AddPacketToInputStream(
+          MP_EXPECT_OK(graph.AddPacketToInputStream(
               absl::StrCat("input_packets_", i), packet));
         }
       }
     }
-    MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
+    MP_ASSERT_OK(graph.CloseAllInputStreams());
-    MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+    MP_ASSERT_OK(graph.WaitUntilDone());
   }
 
   CalculatorGraphConfig graph_config_;
@@ -335,22 +335,22 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
 
   // Start the graph and add five input packets.
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(
+  MP_ASSERT_OK(graph.Initialize(graph_config_,
-      graph_config_, {
+                                {
-                         {"callback_0", Adopt(new auto(wait_0))},
+                                    {"callback_0", Adopt(new auto(wait_0))},
-                         {"callback_1", Adopt(new auto(wait_1))},
+                                    {"callback_1", Adopt(new auto(wait_1))},
-                     }));
+                                }));
-  MEDIAPIPE_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
+  MP_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(10000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(20000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(30000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(40000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(50000)));
 
   // Release the outputs in order 20000, 10000, 30000, 50000, 40000.
@@ -362,8 +362,8 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
   semaphore_0.Release(1);  // 50000
   wait_for([&] { return out_packets.size() >= 3; });
   semaphore_1.Release(1);  // 40000
-  MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 
   // Output packets 10000 and 40000 are superseded and dropped.
   EXPECT_THAT(TimestampValues(out_packets), ElementsAre(20000, 30000, 50000));

mediapipe/calculators/core/matrix_multiply_calculator.cc

@@ -0,0 +1,66 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "Eigen/Core"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+// Perform a (left) matrix multiply.  Meaning (output = A * input)
+// where A is the matrix which is provided as an input side packet.
+//
+// Example config:
+// node {
+//   calculator: "MatrixMultiplyCalculator"
+//   input_stream: "samples"
+//   output_stream: "multiplied_samples"
+//   input_side_packet: "multiplication_matrix"
+// }
+class MatrixMultiplyCalculator : public CalculatorBase {
+ public:
+  MatrixMultiplyCalculator() {}
+  ~MatrixMultiplyCalculator() override {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(MatrixMultiplyCalculator);
+
+// static
+::mediapipe::Status MatrixMultiplyCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->Inputs().Index(0).Set<Matrix>();
+  cc->Outputs().Index(0).Set<Matrix>();
+  cc->InputSidePackets().Index(0).Set<Matrix>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixMultiplyCalculator::Open(CalculatorContext* cc) {
+  // The output is at the same timestamp as the input.
+  cc->SetOffset(TimestampDiff(0));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) {
+  Matrix* multiplied = new Matrix();
+  *multiplied = cc->InputSidePackets().Index(0).Get<Matrix>() *
+                cc->Inputs().Index(0).Get<Matrix>();
+  cc->Outputs().Index(0).Add(multiplied, cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_multiply_calculator_test.cc

@@ -0,0 +1,239 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "Eigen/Core"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+namespace {
+
+// A 3x4 Matrix of random integers in [0,1000).
+const char kMatrixText[] =
+    "rows: 3\n"
+    "cols: 4\n"
+    "packed_data: 387\n"
+    "packed_data: 940\n"
+    "packed_data: 815\n"
+    "packed_data: 825\n"
+    "packed_data: 997\n"
+    "packed_data: 884\n"
+    "packed_data: 419\n"
+    "packed_data: 763\n"
+    "packed_data: 123\n"
+    "packed_data:  30\n"
+    "packed_data: 825\n"
+    "packed_data: 299\n";
+
+// A 4x20 Matrix of random integers in [0,10).
+// Each column of this matrix is a sample.
+const char kSamplesText[] =
+    "rows: 4\n"
+    "cols: 20\n"
+    "packed_data: 7\n"
+    "packed_data: 9\n"
+    "packed_data: 5\n"
+    "packed_data: 9\n"
+    "packed_data: 6\n"
+    "packed_data: 3\n"
+    "packed_data: 0\n"
+    "packed_data: 7\n"
+    "packed_data: 1\n"
+    "packed_data: 3\n"
+    "packed_data: 3\n"
+    "packed_data: 2\n"
+    "packed_data: 4\n"
+    "packed_data: 5\n"
+    "packed_data: 0\n"
+    "packed_data: 4\n"
+    "packed_data: 6\n"
+    "packed_data: 0\n"
+    "packed_data: 1\n"
+    "packed_data: 2\n"
+    "packed_data: 0\n"
+    "packed_data: 2\n"
+    "packed_data: 0\n"
+    "packed_data: 3\n"
+    "packed_data: 1\n"
+    "packed_data: 7\n"
+    "packed_data: 4\n"
+    "packed_data: 9\n"
+    "packed_data: 8\n"
+    "packed_data: 8\n"
+    "packed_data: 6\n"
+    "packed_data: 4\n"
+    "packed_data: 6\n"
+    "packed_data: 8\n"
+    "packed_data: 1\n"
+    "packed_data: 9\n"
+    "packed_data: 7\n"
+    "packed_data: 5\n"
+    "packed_data: 3\n"
+    "packed_data: 5\n"
+    "packed_data: 3\n"
+    "packed_data: 5\n"
+    "packed_data: 7\n"
+    "packed_data: 7\n"
+    "packed_data: 3\n"
+    "packed_data: 3\n"
+    "packed_data: 6\n"
+    "packed_data: 4\n"
+    "packed_data: 7\n"
+    "packed_data: 7\n"
+    "packed_data: 2\n"
+    "packed_data: 5\n"
+    "packed_data: 4\n"
+    "packed_data: 8\n"
+    "packed_data: 1\n"
+    "packed_data: 0\n"
+    "packed_data: 2\n"
+    "packed_data: 0\n"
+    "packed_data: 3\n"
+    "packed_data: 4\n"
+    "packed_data: 6\n"
+    "packed_data: 6\n"
+    "packed_data: 8\n"
+    "packed_data: 5\n"
+    "packed_data: 5\n"
+    "packed_data: 8\n"
+    "packed_data: 9\n"
+    "packed_data: 7\n"
+    "packed_data: 3\n"
+    "packed_data: 7\n"
+    "packed_data: 2\n"
+    "packed_data: 7\n"
+    "packed_data: 8\n"
+    "packed_data: 2\n"
+    "packed_data: 1\n"
+    "packed_data: 1\n"
+    "packed_data: 4\n"
+    "packed_data: 1\n"
+    "packed_data: 1\n"
+    "packed_data: 7\n";
+
+// A 3x20 Matrix of expected values for the result of the matrix multiply
+// computed using R.
+// Each column of this matrix is an expected output.
+const char kExpectedText[] =
+    "rows: 3\n"
+    "cols: 20\n"
+    "packed_data: 12499\n"
+    "packed_data: 26793\n"
+    "packed_data: 16967\n"
+    "packed_data:  5007\n"
+    "packed_data: 14406\n"
+    "packed_data:  9635\n"
+    "packed_data:  4179\n"
+    "packed_data:  7870\n"
+    "packed_data:  4434\n"
+    "packed_data:  5793\n"
+    "packed_data: 12045\n"
+    "packed_data:  8876\n"
+    "packed_data:  2801\n"
+    "packed_data:  8053\n"
+    "packed_data:  5611\n"
+    "packed_data:  1740\n"
+    "packed_data:  4469\n"
+    "packed_data:  2665\n"
+    "packed_data:  8108\n"
+    "packed_data: 18396\n"
+    "packed_data: 10186\n"
+    "packed_data: 12330\n"
+    "packed_data: 23374\n"
+    "packed_data: 15526\n"
+    "packed_data:  9611\n"
+    "packed_data: 21804\n"
+    "packed_data: 14776\n"
+    "packed_data:  8241\n"
+    "packed_data: 17979\n"
+    "packed_data: 11989\n"
+    "packed_data:  8429\n"
+    "packed_data: 18921\n"
+    "packed_data:  9819\n"
+    "packed_data:  6270\n"
+    "packed_data: 13689\n"
+    "packed_data:  7031\n"
+    "packed_data:  9472\n"
+    "packed_data: 19210\n"
+    "packed_data: 13634\n"
+    "packed_data:  8567\n"
+    "packed_data: 12499\n"
+    "packed_data: 10455\n"
+    "packed_data:  2151\n"
+    "packed_data:  7469\n"
+    "packed_data:  3195\n"
+    "packed_data: 10774\n"
+    "packed_data: 21851\n"
+    "packed_data: 12673\n"
+    "packed_data: 12516\n"
+    "packed_data: 25318\n"
+    "packed_data: 14347\n"
+    "packed_data:  7984\n"
+    "packed_data: 17100\n"
+    "packed_data: 10972\n"
+    "packed_data:  5195\n"
+    "packed_data: 11102\n"
+    "packed_data:  8710\n"
+    "packed_data:  3002\n"
+    "packed_data: 11295\n"
+    "packed_data:  6360\n";
+
+// Send a number of samples through the MatrixMultiplyCalculator.
+TEST(MatrixMultiplyCalculatorTest, Multiply) {
+  CalculatorRunner runner("MatrixMultiplyCalculator", "", 1, 1, 1);
+  Matrix* matrix = new Matrix();
+  MatrixFromTextProto(kMatrixText, matrix);
+  runner.MutableSidePackets()->Index(0) = Adopt(matrix);
+
+  Matrix samples;
+  MatrixFromTextProto(kSamplesText, &samples);
+  Matrix expected;
+  MatrixFromTextProto(kExpectedText, &expected);
+  CHECK_EQ(samples.cols(), expected.cols());
+
+  for (int i = 0; i < samples.cols(); ++i) {
+    // Take a column from samples and produce a packet with just that
+    // column in it as an input sample for the calculator.
+    Eigen::MatrixXf* sample = new Eigen::MatrixXf(samples.block(0, i, 4, 1));
+    runner.MutableInputs()->Index(0).packets.push_back(
+        Adopt(sample).At(Timestamp(i)));
+  }
+
+  MP_ASSERT_OK(runner.Run());
+  EXPECT_EQ(runner.MutableInputs()->Index(0).packets.size(),
+            runner.Outputs().Index(0).packets.size());
+
+  int i = 0;
+  for (const Packet& output : runner.Outputs().Index(0).packets) {
+    EXPECT_EQ(Timestamp(i), output.Timestamp());
+    const Eigen::MatrixXf& result = output.Get<Matrix>();
+    ASSERT_EQ(3, result.rows());
+    EXPECT_NEAR((expected.block(0, i, 3, 1) - result).cwiseAbs().sum(), 0.0,
+                1e-5);
+    ++i;
+  }
+  EXPECT_EQ(samples.cols(), i);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_subtract_calculator.cc

@@ -0,0 +1,123 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "Eigen/Core"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Subtract input matrix from the side input matrix and vice versa. The matrices
+// must have the same dimension.
+// Based on the tag (MINUEND vs SUBTRAHEND), the matrices in the input stream
+// and input side packet can be either subtrahend or minuend. The output matrix
+// is generated by performing minuend matrix - subtrahend matrix.
+//
+// Example config:
+// node {
+//   calculator: "MatrixSubtractCalculator"
+//   input_stream: "MINUEND:input_matrix"
+//   input_side_packet: "SUBTRAHEND:side_matrix"
+//   output_stream: "output_matrix"
+// }
+//
+// or
+//
+// node {
+//   calculator: "MatrixSubtractCalculator"
+//   input_stream: "SUBTRAHEND:input_matrix"
+//   input_side_packet: "MINUEND:side_matrix"
+//   output_stream: "output_matrix"
+// }
+class MatrixSubtractCalculator : public CalculatorBase {
+ public:
+  MatrixSubtractCalculator() {}
+  ~MatrixSubtractCalculator() override {}
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  bool subtract_from_input_ = false;
+};
+REGISTER_CALCULATOR(MatrixSubtractCalculator);
+
+// static
+::mediapipe::Status MatrixSubtractCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().NumEntries() != 1 ||
+      cc->InputSidePackets().NumEntries() != 1) {
+    return ::mediapipe::InvalidArgumentError(
+        "MatrixSubtractCalculator only accepts exactly one input stream and "
+        "one "
+        "input side packet");
+  }
+  if (cc->Inputs().HasTag("MINUEND") &&
+      cc->InputSidePackets().HasTag("SUBTRAHEND")) {
+    cc->Inputs().Tag("MINUEND").Set<Matrix>();
+    cc->InputSidePackets().Tag("SUBTRAHEND").Set<Matrix>();
+  } else if (cc->Inputs().HasTag("SUBTRAHEND") &&
+             cc->InputSidePackets().HasTag("MINUEND")) {
+    cc->Inputs().Tag("SUBTRAHEND").Set<Matrix>();
+    cc->InputSidePackets().Tag("MINUEND").Set<Matrix>();
+  } else {
+    return ::mediapipe::InvalidArgumentError(
+        "Must specify exactly one minuend and one subtrahend.");
+  }
+  cc->Outputs().Index(0).Set<Matrix>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixSubtractCalculator::Open(CalculatorContext* cc) {
+  // The output is at the same timestamp as the input.
+  cc->SetOffset(TimestampDiff(0));
+  if (cc->Inputs().HasTag("MINUEND")) {
+    subtract_from_input_ = true;
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) {
+  Matrix* subtracted = new Matrix();
+  if (subtract_from_input_) {
+    const Matrix& input_matrix = cc->Inputs().Tag("MINUEND").Get<Matrix>();
+    const Matrix& side_input_matrix =
+        cc->InputSidePackets().Tag("SUBTRAHEND").Get<Matrix>();
+    if (input_matrix.rows() != side_input_matrix.rows() ||
+        input_matrix.cols() != side_input_matrix.cols()) {
+      return ::mediapipe::InvalidArgumentError(
+          "Input matrix and the input side matrix must have the same "
+          "dimension.");
+    }
+    *subtracted = input_matrix - side_input_matrix;
+  } else {
+    const Matrix& input_matrix = cc->Inputs().Tag("SUBTRAHEND").Get<Matrix>();
+    const Matrix& side_input_matrix =
+        cc->InputSidePackets().Tag("MINUEND").Get<Matrix>();
+    if (input_matrix.rows() != side_input_matrix.rows() ||
+        input_matrix.cols() != side_input_matrix.cols()) {
+      return ::mediapipe::InvalidArgumentError(
+          "Input matrix and the input side matrix must have the same "
+          "dimension.");
+    }
+    *subtracted = side_input_matrix - input_matrix;
+  }
+  cc->Outputs().Index(0).Add(subtracted, cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -0,0 +1,157 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "Eigen/Core"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+namespace {
+
+// A 3x4 Matrix of random integers in [0,1000).
+const char kMatrixText[] =
+    "rows: 3\n"
+    "cols: 4\n"
+    "packed_data: 387\n"
+    "packed_data: 940\n"
+    "packed_data: 815\n"
+    "packed_data: 825\n"
+    "packed_data: 997\n"
+    "packed_data: 884\n"
+    "packed_data: 419\n"
+    "packed_data: 763\n"
+    "packed_data: 123\n"
+    "packed_data:  30\n"
+    "packed_data: 825\n"
+    "packed_data: 299\n";
+
+const char kMatrixText2[] =
+    "rows: 3\n"
+    "cols: 4\n"
+    "packed_data: 388\n"
+    "packed_data: 941\n"
+    "packed_data: 816\n"
+    "packed_data: 826\n"
+    "packed_data: 998\n"
+    "packed_data: 885\n"
+    "packed_data: 420\n"
+    "packed_data: 764\n"
+    "packed_data: 124\n"
+    "packed_data:  31\n"
+    "packed_data: 826\n"
+    "packed_data: 300\n";
+
+TEST(MatrixSubtractCalculatorTest, WrongConfig) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "MatrixSubtractCalculator"
+        input_stream: "input_matrix"
+        input_side_packet: "SUBTRAHEND:side_matrix"
+        input_side_packet: "MINUEND:side_matrix2"
+        output_stream: "output_matrix"
+      )");
+  CalculatorRunner runner(node_config);
+  auto status = runner.Run();
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "only accepts exactly one input stream and one input side packet"));
+}
+
+TEST(MatrixSubtractCalculatorTest, WrongConfig2) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "MatrixSubtractCalculator"
+        input_side_packet: "SUBTRAHEND:side_matrix"
+        input_stream: "SUBTRAHEND:side_matrix2"
+        output_stream: "output_matrix"
+      )");
+  CalculatorRunner runner(node_config);
+  auto status = runner.Run();
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr("specify exactly one minuend and one subtrahend."));
+}
+
+TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "MatrixSubtractCalculator"
+        input_stream: "MINUEND:input_matrix"
+        input_side_packet: "SUBTRAHEND:side_matrix"
+        output_stream: "output_matrix"
+      )");
+  CalculatorRunner runner(node_config);
+  Matrix* side_matrix = new Matrix();
+  MatrixFromTextProto(kMatrixText, side_matrix);
+  runner.MutableSidePackets()->Tag("SUBTRAHEND") = Adopt(side_matrix);
+
+  Matrix* input_matrix = new Matrix();
+  MatrixFromTextProto(kMatrixText2, input_matrix);
+  runner.MutableInputs()->Tag("MINUEND").packets.push_back(
+      Adopt(input_matrix).At(Timestamp(0)));
+
+  MP_ASSERT_OK(runner.Run());
+  EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
+
+  EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());
+  const Eigen::MatrixXf& result =
+      runner.Outputs().Index(0).packets[0].Get<Matrix>();
+  ASSERT_EQ(3, result.rows());
+  ASSERT_EQ(4, result.cols());
+  EXPECT_NEAR(result.sum(), 12, 1e-5);
+}
+
+TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "MatrixSubtractCalculator"
+        input_stream: "SUBTRAHEND:input_matrix"
+        input_side_packet: "MINUEND:side_matrix"
+        output_stream: "output_matrix"
+      )");
+  CalculatorRunner runner(node_config);
+  Matrix* side_matrix = new Matrix();
+  MatrixFromTextProto(kMatrixText, side_matrix);
+  runner.MutableSidePackets()->Tag("MINUEND") = Adopt(side_matrix);
+
+  Matrix* input_matrix = new Matrix();
+  MatrixFromTextProto(kMatrixText2, input_matrix);
+  runner.MutableInputs()
+      ->Tag("SUBTRAHEND")
+      .packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
+
+  MP_ASSERT_OK(runner.Run());
+  EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
+
+  EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());
+  const Eigen::MatrixXf& result =
+      runner.Outputs().Index(0).packets[0].Get<Matrix>();
+  ASSERT_EQ(3, result.rows());
+  ASSERT_EQ(4, result.cols());
+  EXPECT_NEAR(result.sum(), -12, 1e-5);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_to_vector_calculator.cc

@@ -0,0 +1,83 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines MatrixToVectorCalculator.
+#include <math.h>
+
+#include <deque>
+#include <memory>
+#include <string>
+
+#include "Eigen/Core"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// A calculator that converts a Matrix M to a vector containing all the
+// entries of M in column-major order.
+//
+// Example config:
+// node {
+//   calculator: "MatrixToVectorCalculator"
+//   input_stream: "input_matrix"
+//   output_stream: "column_major_vector"
+// }
+class MatrixToVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input Packet containing a Matrix.
+    );
+    cc->Outputs().Index(0).Set<std::vector<float>>(
+        // Output Packet containing a vector, one for each input Packet.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Outputs a packet containing a vector for each input packet.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(MatrixToVectorCalculator);
+
+::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
+  // Inform the framework that we don't alter timestamps.
+  cc->SetOffset(mediapipe::TimestampDiff(0));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
+  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+  auto output = absl::make_unique<std::vector<float>>();
+
+  // The following lines work to convert the Matrix to a vector because Matrix
+  // is an Eigen::MatrixXf and Eigen uses column-major layout by default.
+  output->resize(input.rows() * input.cols());
+  auto output_as_matrix =
+      Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
+  output_as_matrix = input;
+
+  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_to_vector_calculator_test.cc

@@ -0,0 +1,88 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "mediapipe/util/time_series_test_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+namespace {
+
+class MatrixToVectorCalculatorTest
+    : public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
+ protected:
+  void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
+
+  void AppendInput(const std::vector<float>& column_major_data,
+                   int64 timestamp) {
+    ASSERT_EQ(num_input_samples_ * num_input_channels_,
+              column_major_data.size());
+    Eigen::Map<const Matrix> data_map(&column_major_data[0],
+                                      num_input_channels_, num_input_samples_);
+    AppendInputPacket(new Matrix(data_map), timestamp);
+  }
+
+  void SetInputStreamParameters(int num_channels, int num_samples) {
+    num_input_channels_ = num_channels;
+    num_input_samples_ = num_samples;
+    input_sample_rate_ = 100;
+    input_packet_rate_ = 20.0;
+  }
+
+  void SetInputHeader(int num_channels, int num_samples) {
+    SetInputStreamParameters(num_channels, num_samples);
+    FillInputHeader();
+  }
+
+  void CheckOutputPacket(int packet, std::vector<float> expected_vector) {
+    const auto& actual_vector =
+        runner_->Outputs().Index(0).packets[packet].Get<std::vector<float>>();
+    EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector));
+  }
+};
+
+TEST_F(MatrixToVectorCalculatorTest, SingleRow) {
+  InitializeGraph();
+  SetInputHeader(1, 4);  // 1 channel x 4 samples
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0};
+  AppendInput(data_vector, 0);
+  MP_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) {
+  InitializeGraph();
+  SetInputHeader(4, 2);  // 4 channels x 2 samples
+  // Actual data matrix is the transpose of the appearance below.
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0,
+                                           5.0, 6.0, 7.0, 8.0};
+  AppendInput(data_vector, 0);
+
+  MP_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/core/merge_calculator.cc

@@ -0,0 +1,91 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// This calculator takes a set of input streams and combines them into a single
+// output stream. The packets from different streams do not need to contain the
+// same type. If there are packets arriving at the same time from two or more
+// input streams, the packet corresponding to the input stream with the smallest
+// index is passed to the output and the rest are ignored.
+//
+// Example use-case:
+// Suppose we have two (or more) different algorithms for detecting shot
+// boundaries and we need to merge their packets into a single stream. The
+// algorithms may emit shot boundaries at the same time and their output types
+// may not be compatible. Subsequent calculators that process the merged stream
+// may be interested only in the timestamps of the shot boundary packets and so
+// it may not even need to inspect the values stored inside the packets.
+//
+// Example config:
+// node {
+//   calculator: "MergeCalculator"
+//   input_stream: "shot_info1"
+//   input_stream: "shot_info2"
+//   input_stream: "shot_info3"
+//   output_stream: "merged_shot_infos"
+// }
+//
+class MergeCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK_GT(cc->Inputs().NumEntries(), 0)
+        << "Needs at least one input stream";
+    RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+    if (cc->Inputs().NumEntries() == 1) {
+      LOG(WARNING)
+          << "MergeCalculator expects multiple input streams to merge but is "
+             "receiving only one. Make sure the calculator is configured "
+             "correctly or consider removing this calculator to reduce "
+             "unnecessary overhead.";
+    }
+
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      cc->Inputs().Index(i).SetAny();
+    }
+    cc->Outputs().Index(0).SetAny();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(TimestampDiff(0));
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    // Output the packet from the first input stream with a packet ready at this
+    // timestamp.
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      if (!cc->Inputs().Index(i).IsEmpty()) {
+        cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(i).Value());
+        return ::mediapipe::OkStatus();
+      }
+    }
+
+    LOG(WARNING) << "Empty input packets at timestamp "
+                 << cc->InputTimestamp().Value();
+
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(MergeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/merge_calculator_test.cc

@@ -0,0 +1,139 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+// Checks that the calculator fails if no input streams are provided.
+TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "MergeCalculator"
+    output_stream: "merged_output"
+  )"));
+  // Expect calculator to fail.
+  ASSERT_FALSE(runner.Run().ok());
+}
+
+// Checks that the calculator fails with an incorrect number of output streams.
+TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) {
+  CalculatorRunner runner1(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "MergeCalculator"
+    input_stream: "input1"
+    input_stream: "input2"
+  )"));
+  // Expect calculator to fail.
+  EXPECT_FALSE(runner1.Run().ok());
+
+  CalculatorRunner runner2(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "MergeCalculator"
+    input_stream: "input1"
+    input_stream: "input2"
+    output_stream: "output1"
+    output_stream: "output2"
+  )"));
+  // Expect calculator to fail.
+  ASSERT_FALSE(runner2.Run().ok());
+}
+
+// Ensures two streams with differing types can be merged correctly.
+TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
+     TestMergingTwoStreams) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "MergeCalculator"
+    input_stream: "input1"
+    input_stream: "input2"
+    output_stream: "combined_output"
+  )"));
+
+  // input1: integers 10, 20, 30, occurring at times 10, 20, 30.
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(new int(10)).At(Timestamp(10)));
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(new int(20)).At(Timestamp(20)));
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(new int(30)).At(Timestamp(30)));
+  // input2: floats 5.5, 35.5 at times 5, 35.
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(new float(5.5)).At(Timestamp(5)));
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(new float(35.5)).At(Timestamp(35)));
+
+  MP_ASSERT_OK(runner.Run());
+
+  // Expected combined_output: 5.5, 10, 20, 30, 35.5 at times 5, 10, 20, 30, 35.
+  const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(actual_output.size(), 5);
+  EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(5));
+  EXPECT_EQ(actual_output[0].Get<float>(), 5.5);
+
+  EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(10));
+  EXPECT_EQ(actual_output[1].Get<int>(), 10);
+
+  EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(20));
+  EXPECT_EQ(actual_output[2].Get<int>(), 20);
+
+  EXPECT_EQ(actual_output[3].Timestamp(), Timestamp(30));
+  EXPECT_EQ(actual_output[3].Get<int>(), 30);
+
+  EXPECT_EQ(actual_output[4].Timestamp(), Timestamp(35));
+  EXPECT_EQ(actual_output[4].Get<float>(), 35.5);
+}
+
+// Ensures three streams with differing types can be merged correctly.
+TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
+     TestMergingThreeStreams) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "MergeCalculator"
+    input_stream: "input1"
+    input_stream: "input2"
+    input_stream: "input3"
+    output_stream: "combined_output"
+  )"));
+
+  // input1: integer 30 occurring at time 30.
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(new int(30)).At(Timestamp(30)));
+  // input2: float 20.5 occurring at time 20.
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(new float(20.5)).At(Timestamp(20)));
+  // input3: char 'c' occurring at time 10.
+  runner.MutableInputs()->Index(2).packets.push_back(
+      Adopt(new char('c')).At(Timestamp(10)));
+
+  MP_ASSERT_OK(runner.Run());
+
+  // Expected combined_output: 'c', 20.5, 30 at times 10, 20, 30.
+  const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(actual_output.size(), 3);
+  EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(10));
+  EXPECT_EQ(actual_output[0].Get<char>(), 'c');
+
+  EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(20));
+  EXPECT_EQ(actual_output[1].Get<float>(), 20.5);
+
+  EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(30));
+  EXPECT_EQ(actual_output[2].Get<int>(), 30);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/mux_calculator.cc

@@ -51,7 +51,7 @@ class MuxCalculator : public CalculatorBase {
     data_input_base_ = cc->Inputs().GetId("INPUT", 0);
     num_data_inputs_ = cc->Inputs().NumEntries("INPUT");
     output_ = cc->Outputs().GetId("OUTPUT", 0);
-    cc->SetOffset(mediapipe::TimestampDiff(0));
+    cc->SetOffset(TimestampDiff(0));
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/core/packet_cloner_calculator.cc

@@ -19,6 +19,7 @@
 #include <vector>
 
 #include "absl/strings/str_cat.h"
+#include "mediapipe/calculators/core/packet_cloner_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 
 namespace mediapipe {
@@ -39,6 +40,7 @@ namespace mediapipe {
 // }
 //
 // Related:
+//   packet_cloner_calculator.proto: Options for this calculator.
 //   merge_input_streams_calculator.cc: One output stream.
 //   packet_inner_join_calculator.cc: Don't output unless all inputs are new.
 class PacketClonerCalculator : public CalculatorBase {
@@ -54,6 +56,13 @@ class PacketClonerCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) final {
+    // Load options.
+    const auto calculator_options =
+        cc->Options<mediapipe::PacketClonerCalculatorOptions>();
+    output_only_when_all_inputs_received_ =
+        calculator_options.output_only_when_all_inputs_received();
+
+    // Parse input streams.
     tick_signal_index_ = cc->Inputs().NumEntries() - 1;
     current_.resize(tick_signal_index_);
     // Pass along the header for each stream if present.
@@ -73,8 +82,17 @@ class PacketClonerCalculator : public CalculatorBase {
       }
     }
 
-    // Output if the tick signal is non-empty.
+    // Output according to the TICK signal.
     if (!cc->Inputs().Index(tick_signal_index_).Value().IsEmpty()) {
+      if (output_only_when_all_inputs_received_) {
+        // Return if one of the input is null.
+        for (int i = 0; i < tick_signal_index_; ++i) {
+          if (current_[i].IsEmpty()) {
+            return ::mediapipe::OkStatus();
+          }
+        }
+      }
+      // Output each stream.
       for (int i = 0; i < tick_signal_index_; ++i) {
         if (!current_[i].IsEmpty()) {
           cc->Outputs().Index(i).AddPacket(
@@ -91,6 +109,7 @@ class PacketClonerCalculator : public CalculatorBase {
  private:
   std::vector<Packet> current_;
   int tick_signal_index_;
+  bool output_only_when_all_inputs_received_;
 };
 
 REGISTER_CALCULATOR(PacketClonerCalculator);

mediapipe/calculators/core/packet_cloner_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message PacketClonerCalculatorOptions {
+  extend CalculatorOptions {
+    optional PacketClonerCalculatorOptions ext = 258872085;
+  }
+
+  // When true, this calculator will drop received TICK packets if any input
+  // stream hasn't received a packet yet.
+  optional bool output_only_when_all_inputs_received = 1 [default = false];
+}

mediapipe/calculators/core/packet_inner_join_calculator.cc

@@ -0,0 +1,78 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+// Calculator that acts like the SQL query:
+// SELECT *
+// FROM packets_on_stream1 AS packet1
+// INNER JOIN packets_on_stream2 AS packet2
+// ON packet1.timestamp = packet2.timestamp
+//
+// In other words, it only emits and forwards packets if all input streams are
+// not empty.
+//
+// Intended for use with FixedSizeInputStreamHandler.
+//
+// Related:
+//   packet_cloner_calculator.cc: Repeats last-seen packets from empty inputs.
+class PacketInnerJoinCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  int num_streams_;
+};
+
+REGISTER_CALCULATOR(PacketInnerJoinCalculator);
+
+::mediapipe::Status PacketInnerJoinCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(cc->Inputs().NumEntries() == cc->Outputs().NumEntries())
+      << "The number of input and output streams must match.";
+  const int num_streams = cc->Inputs().NumEntries();
+  for (int i = 0; i < num_streams; ++i) {
+    cc->Inputs().Index(i).SetAny();
+    cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) {
+  num_streams_ = cc->Inputs().NumEntries();
+  cc->SetOffset(TimestampDiff(0));
+  return mediapipe::OkStatus();
+}
+
+::mediapipe::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) {
+  for (int i = 0; i < num_streams_; ++i) {
+    if (cc->Inputs().Index(i).Value().IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+  }
+  for (int i = 0; i < num_streams_; ++i) {
+    cc->Outputs().Index(i).AddPacket(cc->Inputs().Index(i).Value());
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/packet_inner_join_calculator_test.cc

@@ -0,0 +1,101 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+namespace {
+
+inline Packet PacketFrom(int i) { return Adopt(new int(i)).At(Timestamp(i)); }
+
+TEST(PacketInnerJoinCalculatorTest, AllMatching) {
+  // Test case.
+  const std::vector<int> packets_on_stream1 = {0, 1, 2, 3};
+  const std::vector<int> packets_on_stream2 = {0, 1, 2, 3};
+  // Run.
+  CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
+  for (int packet_load : packets_on_stream1) {
+    runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
+  }
+  for (int packet_load : packets_on_stream2) {
+    runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
+  }
+  MP_ASSERT_OK(runner.Run());
+  // Check.
+  const std::vector<int> expected = {0, 1, 2, 3};
+  ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());
+  ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size());
+  for (int i = 0; i < expected.size(); ++i) {
+    const Packet packet1 = runner.Outputs().Index(0).packets[i];
+    EXPECT_EQ(expected[i], packet1.Get<int>());
+    EXPECT_EQ(expected[i], packet1.Timestamp().Value());
+    const Packet packet2 = runner.Outputs().Index(1).packets[i];
+    EXPECT_EQ(expected[i], packet2.Get<int>());
+    EXPECT_EQ(expected[i], packet2.Timestamp().Value());
+  }
+}
+
+TEST(PacketInnerJoinCalculatorTest, NoneMatching) {
+  // Test case.
+  const std::vector<int> packets_on_stream1 = {0, 2};
+  const std::vector<int> packets_on_stream2 = {1, 3};
+  // Run.
+  CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
+  for (int packet_load : packets_on_stream1) {
+    runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
+  }
+  for (int packet_load : packets_on_stream2) {
+    runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
+  }
+  MP_ASSERT_OK(runner.Run());
+  // Check.
+  EXPECT_TRUE(runner.Outputs().Index(0).packets.empty());
+  EXPECT_TRUE(runner.Outputs().Index(1).packets.empty());
+}
+
+TEST(PacketInnerJoinCalculatorTest, SomeMatching) {
+  // Test case.
+  const std::vector<int> packets_on_stream1 = {0, 1, 2, 3, 4, 6};
+  const std::vector<int> packets_on_stream2 = {0, 2, 4, 5, 6};
+  // Run.
+  CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0);
+  for (int packet_load : packets_on_stream1) {
+    runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load));
+  }
+  for (int packet_load : packets_on_stream2) {
+    runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
+  }
+  MP_ASSERT_OK(runner.Run());
+  // Check.
+  const std::vector<int> expected = {0, 2, 4, 6};
+  ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());
+  ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size());
+  for (int i = 0; i < expected.size(); ++i) {
+    const Packet packet1 = runner.Outputs().Index(0).packets[i];
+    EXPECT_EQ(expected[i], packet1.Get<int>());
+    EXPECT_EQ(expected[i], packet1.Timestamp().Value());
+    const Packet packet2 = runner.Outputs().Index(1).packets[i];
+    EXPECT_EQ(expected[i], packet2.Get<int>());
+    EXPECT_EQ(expected[i], packet2.Timestamp().Value());
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -74,6 +74,12 @@ class PacketResamplerCalculator : public CalculatorBase {
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestampWithJitter();
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestampWithJitter();
+
   // Logic for Process() when jitter_ != 0.0.
   ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
 
@@ -233,6 +239,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
       << Timestamp::kTimestampUnitsPerSecond;
 
   frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
+
   video_header_.frame_rate = frame_rate_;
 
   if (resampler_options.output_header() !=
@@ -287,14 +294,25 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
     }
   }
   if (jitter_ != 0.0 && random_ != nullptr) {
-    RETURN_IF_ERROR(ProcessWithJitter(cc));
+    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
   } else {
-    RETURN_IF_ERROR(ProcessWithoutJitter(cc));
+    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
   }
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
   return ::mediapipe::OkStatus();
 }
 
+void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_ =
+      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
+}
+
+void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
+  next_output_timestamp_ +=
+      frame_time_usec_ *
+      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
+}
+
 ::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
     CalculatorContext* cc) {
   RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
@@ -302,8 +320,13 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 
   if (first_timestamp_ == Timestamp::Unset()) {
     first_timestamp_ = cc->InputTimestamp();
-    next_output_timestamp_ =
+    InitializeNextOutputTimestampWithJitter();
-        first_timestamp_ + frame_time_usec_ * random_->RandFloat();
+    if (first_timestamp_ == next_output_timestamp_) {
+      OutputWithinLimits(
+          cc,
+          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
     return ::mediapipe::OkStatus();
   }
 
@@ -322,9 +345,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
                               ? last_packet_
                               : cc->Inputs().Get(input_data_id_).Value())
                              .At(next_output_timestamp_));
-  next_output_timestamp_ +=
+  UpdateNextOutputTimestampWithJitter();
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -103,7 +103,7 @@ TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
   }
 }
 
@@ -114,7 +114,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0}, {0});
   }
 
@@ -124,7 +124,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({1000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({1000}, {1000});
   }
 
@@ -134,7 +134,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({16668});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({16668}, {16668});
   }
 }
@@ -146,7 +146,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16666});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0}, {0});
   }
 
@@ -156,7 +156,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 16667}, {0, 33333});
   }
 
@@ -166,7 +166,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 49999});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 49999}, {0, 33333});
   }
 
@@ -176,7 +176,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 50000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 0, 50000}, {0, 33333, 66667});
   }
 
@@ -186,7 +186,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({2000, 118666});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({2000, 2000, 2000, 118666},
                                  {2000, 35333, 68667, 102000});
   }
@@ -197,7 +197,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepoints) {
       "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
       "{frame_rate:30}");
   runner.SetInput({0, 33333, 66667, 100000, 133333, 166667, 200000});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   runner.CheckOutputTimestamps(
       {0, 33333, 66667, 100000, 133333, 166667, 200000},
       {0, 33333, 66667, 100000, 133333, 166667, 200000});
@@ -210,7 +210,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriods) {
       "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
       "{frame_rate:30}");
   runner.SetInput({0, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   runner.CheckOutputTimestamps({0, 33300, 66600}, {0, 33333, 66667});
 }
 
@@ -222,7 +222,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 5000, 16666, 83334});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 16666, 16666, 83334},
                                  {0, 33333, 66667, 100000});
   }
@@ -232,7 +232,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16666, 16667, 25000, 33000, 35000, 135000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 33000, 35000, 35000, 135000},
                                  {0, 33333, 66667, 100000, 133333});
   }
@@ -242,7 +242,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -255,7 +255,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:500000}");
     runner.SetInput({0, 10, 13});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 0, 0, 0, 0, 10, 10, 13},
                                  {0, 2, 4, 6, 8, 10, 12, 14});
   }
@@ -266,7 +266,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:1000000}");
     runner.SetInput({0, 10, 13});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps(
         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 13},
         {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13});
@@ -280,7 +280,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-200, -20, 16466});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-200}, {-200});
   }
 
@@ -290,7 +290,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-200, -20, 16467});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-200, 16467}, {-200, 33133});
   }
 
@@ -300,7 +300,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-500, 66667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-500, -500, 66667}, {-500, 32833, 66167});
   }
 
@@ -310,7 +310,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-50000, -33334, 33334});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-50000, -33334, -33334, 33334},
                                  {-50000, -16667, 16667, 50000});
   }
@@ -323,7 +323,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:50}");
     runner.SetInput({0, 9999, 29999});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 29999}, {0, 20000});
   }
 
@@ -333,7 +333,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:50}");
     runner.SetInput({0, 10000, 50000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 10000, 10000, 50000},
                                  {0, 20000, 40000, 60000});
   }
@@ -347,7 +347,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
     runner.SetInput({0, 10000, 30000, 50000, 60000});
     runner.SetVideoHeader(50.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 10000, 30000, 60000},
                                  {0, 20000, 40000, 60000});
     runner.CheckVideoHeader(50.0);
@@ -360,7 +360,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
     runner.SetInput({0, 5000, 10010, 15001, 19990});
     runner.SetVideoHeader(200.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
     runner.CheckVideoHeader(50.0);
   }
@@ -372,7 +372,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:PASS_HEADER}");
     runner.SetInput({0, 5000, 10010, 15001, 19990});
     runner.SetVideoHeader(200.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
     runner.CheckVideoHeader(200.0);
   }
@@ -404,7 +404,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
       ->Tag("VIDEO_HEADER")
       .packets.push_back(
           Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream()));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size());
   EXPECT_EQ(Timestamp::PreStream(),
@@ -424,7 +424,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithoutRound) {
         frame_rate: 1
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1333333 is not emitted as 2000000, because it does not round to 2000000.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -435,7 +435,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithRound) {
         frame_rate: 1
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1666667 is emitted as 2000000, because it rounds to 2000000.
   runner.CheckOutputTimestamps({0, 1000000, 1666667}, {0, 1000000, 2000000});
 }
@@ -447,7 +447,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithoutRound) {
         flush_last_packet: false
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1333333 is not emitted no matter what; see FlushLastPacketWithoutRound.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -459,7 +459,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithRound) {
         flush_last_packet: false
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1666667 is not emitted due to flush_last_packet: false.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -473,7 +473,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({33111, 66667, 100000, 133333, 166667, 200000},
                                  {33111, 66444, 99778, 133111, 166444, 199778});
   }
@@ -484,7 +484,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps(
         {33111, 66667, 100000, 133333, 166667, 200000},
         {33333, 66666, 100000, 133333, 166666, 200000});
@@ -499,7 +499,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 33300, 66600}, {-222, 33111, 66445});
   }
   {
@@ -509,7 +509,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "{frame_rate:30 "
         "base_timestamp:900011}");
     runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 33300, 66600}, {11, 33344, 66678});
   }
   {
@@ -521,7 +521,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "base_timestamp:11}");
     runner.SetInput(
         {899888, 916666, 916667, 920000, 933300, 949999, 950000, 966600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({899888, 933300, 966600},
                                  {900011, 933344, 966678});
   }
@@ -536,7 +536,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {-222, 33111, 66445, 99778, 133111, 166445});
   }
@@ -547,7 +547,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -565,7 +565,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({66667, 100020, 133333, 166667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({66667, 100020, 133333, 166667},
                                  {66667, 100000, 133334, 166667});
   }
@@ -582,7 +582,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({100020, 133333, 166667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({100020, 133333, 166667},
                                  {100000, 133333, 166667});
   }
@@ -596,7 +596,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -609,7 +609,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "start_time:40000 "
         "end_time:160000}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({49999, 49999, 49999},
                                  {66667, 100000, 133333});
   }
@@ -624,7 +624,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "end_time:160000 "
         "round_limits:true}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({32000, 49999, 49999, 49999, 150000},
                                  {33333, 66667, 100000, 133333, 166667});
   }
@@ -654,7 +654,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
               })"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
   }
   {
@@ -670,7 +670,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
   }
 }

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -74,10 +74,6 @@ class PreviousLoopbackCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) final {
-    Packet& main_packet = cc->Inputs().Get(main_id_).Value();
-    if (!main_packet.IsEmpty()) {
-      main_ts_.push_back(main_packet.Timestamp());
-    }
     Packet& loopback_packet = cc->Inputs().Get(loop_id_).Value();
     if (!loopback_packet.IsEmpty()) {
       loopback_packets_.push_back(loopback_packet);
@@ -87,6 +83,23 @@ class PreviousLoopbackCalculator : public CalculatorBase {
       }
     }
 
+    Packet& main_packet = cc->Inputs().Get(main_id_).Value();
+    if (!main_packet.IsEmpty()) {
+      main_ts_.push_back(main_packet.Timestamp());
+
+      // In case of an empty "LOOP" input, truncate timestamp is set to the
+      // lowest possible timestamp for a successive non-empty "LOOP" input. This
+      // truncates main_ts_ as soon as possible, and produces the highest legal
+      // output timestamp bound.
+      if (loopback_packet.IsEmpty() &&
+          loopback_packet.Timestamp() != Timestamp::Unstarted()) {
+        while (!main_ts_.empty() &&
+               main_ts_.front() <= loopback_packet.Timestamp() + 1) {
+          main_ts_.pop_front();
+        }
+      }
+    }
+
     while (!main_ts_.empty() && !loopback_packets_.empty()) {
       Timestamp main_timestamp = main_ts_.front();
       main_ts_.pop_front();
@@ -102,6 +115,12 @@ class PreviousLoopbackCalculator : public CalculatorBase {
         cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
       }
     }
+    if (!main_ts_.empty()) {
+      cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
+    }
+    if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
+      cc->Outputs().Get(loop_out_id_).Close();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -74,11 +74,11 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   tool::AddVectorSink("pair", &graph_config_, &in_prev);
 
   CalculatorGraph graph_;
-  MEDIAPIPE_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   auto pair_values = [](const Packet& packet) {
@@ -89,22 +89,172 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   };
 
   send_packet("in", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));
 
   send_packet("in", 5);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
 
   send_packet("in", 15);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
+// A Calculator that outputs a summary packet in CalculatorBase::Close().
+class PacketOnCloseCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    sum_ += cc->Inputs().Index(0).Value().Get<int>();
+    cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Close(CalculatorContext* cc) final {
+    cc->Outputs().Index(0).AddPacket(
+        MakePacket<int>(sum_).At(Timestamp::Max()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  int sum_ = 0;
+};
+REGISTER_CALCULATOR(PacketOnCloseCalculator);
+
+// Demonstrates that all ouput and input streams in PreviousLoopbackCalculator
+// will close as expected when all graph input streams are closed.
+TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
+  std::vector<Packet> outputs;
+  CalculatorGraphConfig graph_config_ =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: 'in'
+        node {
+          calculator: 'PreviousLoopbackCalculator'
+          input_stream: 'MAIN:in'
+          input_stream: 'LOOP:out'
+          input_stream_info: { tag_index: 'LOOP' back_edge: true }
+          output_stream: 'PREV_LOOP:previous'
+        }
+        # This calculator synchronizes its inputs as normal, so it is used
+        # to check that both "in" and "previous" are ready.
+        node {
+          calculator: 'PassThroughCalculator'
+          input_stream: 'in'
+          input_stream: 'previous'
+          output_stream: 'out'
+          output_stream: 'previous2'
+        }
+        node {
+          calculator: 'PacketOnCloseCalculator'
+          input_stream: 'out'
+          output_stream: 'close_out'
+        }
+      )");
+  tool::AddVectorSink("close_out", &graph_config_, &outputs);
+
+  CalculatorGraph graph_;
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.StartRun({}));
+
+  auto send_packet = [&graph_](const std::string& input_name, int n) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        input_name, MakePacket<int>(n).At(Timestamp(n))));
+  };
+
+  send_packet("in", 1);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));
+
+  send_packet("in", 5);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5}));
+
+  send_packet("in", 15);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5, 15}));
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs),
+            (std::vector<int64>{1, 5, 15, Timestamp::Max().Value()}));
+
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
+// Demonstrates that downstream calculators won't be blocked by
+// always-empty-LOOP-stream.
+TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
+  std::vector<Packet> outputs;
+  CalculatorGraphConfig graph_config_ =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: 'in'
+        node {
+          calculator: 'PreviousLoopbackCalculator'
+          input_stream: 'MAIN:in'
+          input_stream: 'LOOP:previous'
+          input_stream_info: { tag_index: 'LOOP' back_edge: true }
+          output_stream: 'PREV_LOOP:previous'
+        }
+        # This calculator synchronizes its inputs as normal, so it is used
+        # to check that both "in" and "previous" are ready.
+        node {
+          calculator: 'PassThroughCalculator'
+          input_stream: 'in'
+          input_stream: 'previous'
+          output_stream: 'out'
+          output_stream: 'previous2'
+        }
+        node {
+          calculator: 'PacketOnCloseCalculator'
+          input_stream: 'out'
+          output_stream: 'close_out'
+        }
+      )");
+  tool::AddVectorSink("close_out", &graph_config_, &outputs);
+
+  CalculatorGraph graph_;
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.StartRun({}));
+
+  auto send_packet = [&graph_](const std::string& input_name, int n) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        input_name, MakePacket<int>(n).At(Timestamp(n))));
+  };
+
+  send_packet("in", 0);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{0}));
+
+  for (int main_ts = 1; main_ts < 50; ++main_ts) {
+    send_packet("in", main_ts);
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
+    std::vector<int64> ts_values = TimestampValues(outputs);
+    EXPECT_EQ(ts_values.size(), main_ts);
+    for (int j = 0; j < main_ts; ++j) {
+      CHECK_EQ(ts_values[j], j);
+    }
+  }
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
 }  // anonymous namespace

mediapipe/calculators/core/quantize_float_vector_calculator.cc

@@ -0,0 +1,102 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cfloat>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/calculators/core/quantize_float_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/status.h"
+
+// Quantizes a vector of floats to a std::string so that each float becomes a
+// byte in the [0, 255] range. Any value above max_quantized_value or below
+// min_quantized_value will be saturated to '/xFF' or '/0'.
+//
+// Example config:
+//   node {
+//     calculator: "QuantizeFloatVectorCalculator"
+//     input_stream: "FLOAT_VECTOR:float_vector"
+//     output_stream: "ENCODED:encoded"
+//     options {
+//       [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+//         max_quantized_value: 64
+//         min_quantized_value: -64
+//       }
+//     }
+//   }
+namespace mediapipe {
+
+class QuantizeFloatVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
+    cc->Outputs().Tag("ENCODED").Set<std::string>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    const auto options =
+        cc->Options<::mediapipe::QuantizeFloatVectorCalculatorOptions>();
+    if (!options.has_max_quantized_value() ||
+        !options.has_min_quantized_value()) {
+      return ::mediapipe::InvalidArgumentError(
+          "Both max_quantized_value and min_quantized_value must be provided "
+          "in QuantizeFloatVectorCalculatorOptions.");
+    }
+    max_quantized_value_ = options.max_quantized_value();
+    min_quantized_value_ = options.min_quantized_value();
+    if (max_quantized_value_ < min_quantized_value_ + FLT_EPSILON) {
+      return ::mediapipe::InvalidArgumentError(
+          "max_quantized_value must be greater than min_quantized_value.");
+    }
+    range_ = max_quantized_value_ - min_quantized_value_;
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    const std::vector<float>& float_vector =
+        cc->Inputs().Tag("FLOAT_VECTOR").Value().Get<std::vector<float>>();
+    int feature_size = float_vector.size();
+    std::string encoded_features;
+    encoded_features.reserve(feature_size);
+    for (int i = 0; i < feature_size; i++) {
+      float old_value = float_vector[i];
+      if (old_value < min_quantized_value_) {
+        old_value = min_quantized_value_;
+      }
+      if (old_value > max_quantized_value_) {
+        old_value = max_quantized_value_;
+      }
+      unsigned char encoded = static_cast<unsigned char>(
+          (old_value - min_quantized_value_) * (255.0 / range_));
+      encoded_features += encoded;
+    }
+    cc->Outputs().Tag("ENCODED").AddPacket(
+        MakePacket<std::string>(encoded_features).At(cc->InputTimestamp()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float max_quantized_value_;
+  float min_quantized_value_;
+  float range_;
+};
+
+REGISTER_CALCULATOR(QuantizeFloatVectorCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/quantize_float_vector_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message QuantizeFloatVectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional QuantizeFloatVectorCalculatorOptions ext = 259848061;
+  }
+
+  optional float max_quantized_value = 1;
+  optional float min_quantized_value = 2;
+}

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -0,0 +1,204 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            min_quantized_value: 1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> empty_vector;
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "Both max_quantized_value and min_quantized_value must be provided"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            max_quantized_value: -1
+            min_quantized_value: 1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> empty_vector;
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            max_quantized_value: 1
+            min_quantized_value: 1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> empty_vector;
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            max_quantized_value: 1
+            min_quantized_value: -1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> empty_vector;
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_TRUE(outputs[0].Get<std::string>().empty());
+  EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            max_quantized_value: 64
+            min_quantized_value: -64
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f};
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  EXPECT_EQ(1, outputs.size());
+  const std::string& result = outputs[0].Get<std::string>();
+  ASSERT_FALSE(result.empty());
+  EXPECT_EQ(5, result.size());
+  // 127
+  EXPECT_EQ('\x7F', result.c_str()[0]);
+  // 0
+  EXPECT_EQ('\0', result.c_str()[1]);
+  // 255
+  EXPECT_EQ('\xFF', result.c_str()[2]);
+  // 63
+  EXPECT_EQ('\x3F', result.c_str()[3]);
+  // 191
+  EXPECT_EQ('\xBF', result.c_str()[4]);
+  EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "QuantizeFloatVectorCalculator"
+        input_stream: "FLOAT_VECTOR:float_vector"
+        output_stream: "ENCODED:encoded"
+        options {
+          [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: {
+            max_quantized_value: 64
+            min_quantized_value: -64
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::vector<float> vector = {-65.0f, 65.0f};
+  runner.MutableInputs()
+      ->Tag("FLOAT_VECTOR")
+      .packets.push_back(
+          MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
+  EXPECT_EQ(1, outputs.size());
+  const std::string& result = outputs[0].Get<std::string>();
+  ASSERT_FALSE(result.empty());
+  EXPECT_EQ(2, result.size());
+  // 0
+  EXPECT_EQ('\0', result.c_str()[0]);
+  // 255
+  EXPECT_EQ('\xFF', result.c_str()[1]);
+  EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/sequence_shift_calculator.cc

@@ -0,0 +1,114 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <deque>
+
+#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+
+namespace mediapipe {
+
+// A Calculator that shifts the timestamps of packets along a stream. Packets on
+// the input stream are output with a timestamp of the packet given by packet
+// offset. That is, packet[i] is output with the timestamp of
+// packet[i + packet_offset]. Packet offset can be either positive or negative.
+// If packet_offset is -n, the first n packets will be dropped. If packet offset
+// is n, the final n packets will be dropped. For example, with a packet_offset
+// of -1, the first packet on the stream will be dropped, the second will be
+// output with the timestamp of the first, the third with the timestamp of the
+// second, and so on.
+class SequenceShiftCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).SetAny();
+    cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  // Reads from options to set cache_size_ and packet_offset_.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  // A positive offset means we want a packet to be output with the timestamp of
+  // a later packet. Stores packets waiting for their output timestamps and
+  // outputs a single packet when the cache fills.
+  void ProcessPositiveOffset(CalculatorContext* cc);
+
+  // A negative offset means we want a packet to be output with the timestamp of
+  // an earlier packet. Stores timestamps waiting for the corresponding input
+  // packet and outputs a single packet when the cache fills.
+  void ProcessNegativeOffset(CalculatorContext* cc);
+
+  // Storage for packets waiting to be output when packet_offset > 0. When cache
+  // is full, oldest packet is output with current timestamp.
+  std::deque<Packet> packet_cache_;
+
+  // Storage for previous timestamps used when packet_offset < 0. When cache is
+  // full, oldest timestamp is used for current packet.
+  std::deque<Timestamp> timestamp_cache_;
+
+  // Copied from corresponding field in options.
+  int packet_offset_;
+  // The number of packets or timestamps we need to store to output packet[i] at
+  // the timestamp of packet[i + packet_offset]; equal to abs(packet_offset).
+  int cache_size_;
+};
+REGISTER_CALCULATOR(SequenceShiftCalculator);
+
+::mediapipe::Status SequenceShiftCalculator::Open(CalculatorContext* cc) {
+  packet_offset_ =
+      cc->Options<mediapipe::SequenceShiftCalculatorOptions>().packet_offset();
+  cache_size_ = abs(packet_offset_);
+  // An offset of zero is a no-op, but someone might still request it.
+  if (packet_offset_ == 0) {
+    cc->Outputs().Index(0).SetOffset(0);
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SequenceShiftCalculator::Process(CalculatorContext* cc) {
+  if (packet_offset_ > 0) {
+    ProcessPositiveOffset(cc);
+  } else if (packet_offset_ < 0) {
+    ProcessNegativeOffset(cc);
+  } else {
+    cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
+  }
+  return ::mediapipe::OkStatus();
+}
+
+void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
+  if (packet_cache_.size() >= cache_size_) {
+    // Ready to output oldest packet with current timestamp.
+    cc->Outputs().Index(0).AddPacket(
+        packet_cache_.front().At(cc->InputTimestamp()));
+    packet_cache_.pop_front();
+  }
+  // Store current packet for later output.
+  packet_cache_.push_back(cc->Inputs().Index(0).Value());
+}
+
+void SequenceShiftCalculator::ProcessNegativeOffset(CalculatorContext* cc) {
+  if (timestamp_cache_.size() >= cache_size_) {
+    // Ready to output current packet with oldest timestamp.
+    cc->Outputs().Index(0).AddPacket(
+        cc->Inputs().Index(0).Value().At(timestamp_cache_.front()));
+    timestamp_cache_.pop_front();
+  }
+  // Store current timestamp for use by a future packet.
+  timestamp_cache_.push_back(cc->InputTimestamp());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/sequence_shift_calculator.proto

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message SequenceShiftCalculatorOptions {
+  extend CalculatorOptions {
+    optional SequenceShiftCalculatorOptions ext = 107633927;
+  }
+  optional int32 packet_offset = 1 [default = -1];
+}

mediapipe/calculators/core/sequence_shift_calculator_test.cc

@@ -0,0 +1,104 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+
+namespace mediapipe {
+
+namespace {
+
+// Adds packets containing integers equal to their original timestamp.
+void AddPackets(CalculatorRunner* runner) {
+  for (int i = 0; i < 10; ++i) {
+    runner->MutableInputs()->Index(0).packets.push_back(
+        Adopt(new int(i)).At(Timestamp(i)));
+  }
+}
+
+// Zero shift is a no-op (output input[i] at timestamp[i]). Input and output
+// streams should be identical.
+TEST(SequenceShiftCalculatorTest, ZeroShift) {
+  CalculatorRunner runner(
+      "SequenceShiftCalculator",
+      "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 0 }", 1,
+      1, 0);
+  AddPackets(&runner);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& input_packets =
+      runner.MutableInputs()->Index(0).packets;
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(10, input_packets.size());
+  ASSERT_EQ(input_packets.size(), output_packets.size());
+  for (int i = 0; i < output_packets.size(); ++i) {
+    // Make sure the contents are as expected.
+    EXPECT_EQ(input_packets[i].Get<int>(), output_packets[i].Get<int>());
+    EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
+  }
+}
+
+// Tests shifting by three packets, i.e., output input[i] with the timestamp of
+// input[i + 3].
+TEST(SequenceShiftCalculatorTest, PositiveShift) {
+  CalculatorRunner runner(
+      "SequenceShiftCalculator",
+      "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 3 }", 1,
+      1, 0);
+  AddPackets(&runner);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& input_packets =
+      runner.MutableInputs()->Index(0).packets;
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(10, input_packets.size());
+  // input_packet[i] should be output with the timestamp of input_packet[i + 3].
+  // The last 3 packets are dropped.
+  ASSERT_EQ(7, output_packets.size());
+  for (int i = 0; i < output_packets.size(); ++i) {
+    // Make sure the contents are as expected.
+    EXPECT_EQ(input_packets[i].Get<int>(), output_packets[i].Get<int>());
+    // Make sure the timestamps are shifted as expected.
+    EXPECT_EQ(input_packets[i + 3].Timestamp(), output_packets[i].Timestamp());
+  }
+}
+
+// Tests shifting by -2, i.e., output input[i] with timestamp[i - 2]. The first
+// two packets should be dropped.
+TEST(SequenceShiftCalculatorTest, NegativeShift) {
+  CalculatorRunner runner(
+      "SequenceShiftCalculator",
+      "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: -2 }",
+      1, 1, 0);
+  AddPackets(&runner);
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& input_packets =
+      runner.MutableInputs()->Index(0).packets;
+  const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
+  ASSERT_EQ(10, input_packets.size());
+  // Input packet[i] should be output with the timestamp of input packet[i - 2].
+  // The first two packets are dropped. This means timestamps match between
+  // input and output packets, but the data in the output packets come from
+  // input_packets[i + 2].
+  ASSERT_EQ(8, output_packets.size());
+  for (int i = 0; i < output_packets.size(); ++i) {
+    EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp());
+    EXPECT_EQ(input_packets[i + 2].Get<int>(), output_packets[i].Get<int>());
+  }
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/core/side_packet_to_stream_calculator.cc

@@ -0,0 +1,83 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+using mediapipe::PacketTypeSet;
+using mediapipe::Timestamp;
+
+namespace {
+static std::map<std::string, Timestamp>* kTimestampMap = []() {
+  auto* res = new std::map<std::string, Timestamp>();
+  res->emplace("AT_PRESTREAM", Timestamp::PreStream());
+  res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
+  res->emplace("AT_ZERO", Timestamp(0));
+  return res;
+}();
+
+}  // namespace
+
+// Outputs the single input_side_packet at the timestamp specified in the
+// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
+class SidePacketToStreamCalculator : public CalculatorBase {
+ public:
+  SidePacketToStreamCalculator() = default;
+  ~SidePacketToStreamCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(SidePacketToStreamCalculator);
+
+::mediapipe::Status SidePacketToStreamCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->InputSidePackets().Index(0).SetAny();
+
+  std::set<std::string> tags = cc->Outputs().GetTags();
+  RET_CHECK_EQ(tags.size(), 1);
+
+  RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
+  cc->Outputs().Tag(*tags.begin()).SetAny();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SidePacketToStreamCalculator::Process(
+    CalculatorContext* cc) {
+  return mediapipe::tool::StatusStop();
+}
+
+::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
+  std::set<std::string> tags = cc->Outputs().GetTags();
+  RET_CHECK_EQ(tags.size(), 1);
+  const std::string& tag = *tags.begin();
+  RET_CHECK_EQ(kTimestampMap->count(tag), 1);
+  cc->Outputs().Tag(tag).AddPacket(
+      cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -0,0 +1,44 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/split_vector_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "SplitTfLiteTensorVectorCalculator"
+//   input_stream: "tflitetensor_vector"
+//   output_stream: "tflitetensor_vector_range_0"
+//   output_stream: "tflitetensor_vector_range_1"
+//   options {
+//     [mediapipe.SplitVectorCalculatorOptions.ext] {
+//       ranges: { begin: 0 end: 1 }
+//       ranges: { begin: 1 end: 2 }
+//       element_only: false
+//     }
+//   }
+// }
+typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
+REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
+
+typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
+    SplitLandmarkVectorCalculator;
+REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
+}  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.h

@@ -0,0 +1,162 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_
+
+#include <vector>
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+#include "tensorflow/lite/error_reporter.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+
+namespace mediapipe {
+
+// Splits an input packet with std::vector<T> into multiple std::vector<T>
+// output packets using the [begin, end) ranges specified in
+// SplitVectorCalculatorOptions. If the option "element_only" is set to true,
+// all ranges should be of size 1 and all outputs will be elements of type T. If
+// "element_only" is false, ranges can be non-zero in size and all outputs will
+// be of type std::vector<T>. If the option "combine_outputs" is set to true,
+// only one output stream can be specified and all ranges of elements will be
+// combined into one vector.
+// To use this class for a particular type T, register a calculator using
+// SplitVectorCalculator<T>.
+template <typename T>
+class SplitVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() != 0);
+
+    cc->Inputs().Index(0).Set<std::vector<T>>();
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    if (options.combine_outputs()) {
+      RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
+      cc->Outputs().Index(0).Set<std::vector<T>>();
+      for (int i = 0; i < options.ranges_size() - 1; ++i) {
+        for (int j = i + 1; j < options.ranges_size(); ++j) {
+          const auto& range_0 = options.ranges(i);
+          const auto& range_1 = options.ranges(j);
+          if ((range_0.begin() >= range_1.begin() &&
+               range_0.begin() < range_1.end()) ||
+              (range_1.begin() >= range_0.begin() &&
+               range_1.begin() < range_0.end())) {
+            return ::mediapipe::InvalidArgumentError(
+                "Ranges must be non-overlapping when using combine_outputs "
+                "option.");
+          }
+        }
+      }
+    } else {
+      if (cc->Outputs().NumEntries() != options.ranges_size()) {
+        return ::mediapipe::InvalidArgumentError(
+            "The number of output streams should match the number of ranges "
+            "specified in the CalculatorOptions.");
+      }
+
+      // Set the output types for each output stream.
+      for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+        if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+            options.ranges(i).begin() >= options.ranges(i).end()) {
+          return ::mediapipe::InvalidArgumentError(
+              "Indices should be non-negative and begin index should be less "
+              "than the end index.");
+        }
+        if (options.element_only()) {
+          if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+            return ::mediapipe::InvalidArgumentError(
+                "Since element_only is true, all ranges should be of size 1.");
+          }
+          cc->Outputs().Index(i).Set<T>();
+        } else {
+          cc->Outputs().Index(i).Set<std::vector<T>>();
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    const auto& options =
+        cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
+
+    element_only_ = options.element_only();
+    combine_outputs_ = options.combine_outputs();
+
+    for (const auto& range : options.ranges()) {
+      ranges_.push_back({range.begin(), range.end()});
+      max_range_end_ = std::max(max_range_end_, range.end());
+      total_elements_ += range.end() - range.begin();
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const auto& input = cc->Inputs().Index(0).Get<std::vector<T>>();
+    RET_CHECK_GE(input.size(), max_range_end_);
+
+    if (combine_outputs_) {
+      auto output = absl::make_unique<std::vector<T>>();
+      output->reserve(total_elements_);
+      for (int i = 0; i < ranges_.size(); ++i) {
+        auto elements = absl::make_unique<std::vector<T>>(
+            input.begin() + ranges_[i].first,
+            input.begin() + ranges_[i].second);
+        output->insert(output->end(), elements->begin(), elements->end());
+      }
+      cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<T>(input[ranges_[i].first]).At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          auto output = absl::make_unique<std::vector<T>>(
+              input.begin() + ranges_[i].first,
+              input.begin() + ranges_[i].second);
+          cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
+        }
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::vector<std::pair<int32, int32>> ranges_;
+  int32 max_range_end_ = -1;
+  int32 total_elements_ = 0;
+  bool element_only_ = false;
+  bool combine_outputs_ = false;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_

mediapipe/calculators/core/split_vector_calculator.proto

@@ -0,0 +1,43 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+// A Range {begin, end} specifies beginning ane ending indices to splice a
+// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
+// begin index inclusive and end index exclusive.
+message Range {
+  optional int32 begin = 1;
+  optional int32 end = 2;
+}
+
+message SplitVectorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional SplitVectorCalculatorOptions ext = 259438222;
+  }
+
+  repeated Range ranges = 1;
+
+  // Specify if single element ranges should be outputted as std::vector<T> or
+  // just element of type T. By default, if a range specifies only one element,
+  // it is outputted as an std::vector<T>.
+  optional bool element_only = 2 [default = false];
+
+  // Combines output elements to one vector.
+  optional bool combine_outputs = 3 [default = false];
+}

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -0,0 +1,455 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/split_vector_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+#include "mediapipe/framework/tool/validate_type.h"
+#include "tensorflow/lite/error_reporter.h"
+#include "tensorflow/lite/interpreter.h"
+#include "tensorflow/lite/kernels/register.h"
+#include "tensorflow/lite/model.h"
+
+namespace mediapipe {
+
+using ::tflite::Interpreter;
+
+const int width = 1;
+const int height = 1;
+const int channels = 1;
+
+class SplitTfLiteTensorVectorCalculatorTest : public ::testing::Test {
+ protected:
+  void TearDown() {
+    // Note: Since the pointers contained in this vector will be cleaned up by
+    // the interpreter, only ensure that the vector is cleaned up for the next
+    // test.
+    input_buffers_.clear();
+  }
+
+  void PrepareTfLiteTensorVector(int vector_size) {
+    ASSERT_NE(interpreter_, nullptr);
+
+    // Prepare input tensors.
+    std::vector<int> indices(vector_size);
+    for (int i = 0; i < vector_size; ++i) {
+      indices[i] = i;
+    }
+    interpreter_->AddTensors(vector_size);
+    interpreter_->SetInputs(indices);
+
+    input_vec_ = absl::make_unique<std::vector<TfLiteTensor>>();
+    for (int i = 0; i < vector_size; ++i) {
+      interpreter_->SetTensorParametersReadWrite(i, kTfLiteFloat32, "", {3},
+                                                 TfLiteQuantization());
+      const int tensor_index = interpreter_->inputs()[i];
+      interpreter_->ResizeInputTensor(tensor_index, {width, height, channels});
+    }
+
+    interpreter_->AllocateTensors();
+
+    // Save the tensor buffer pointers for comparison after the graph runs.
+    input_buffers_ = std::vector<float*>(vector_size);
+    for (int i = 0; i < vector_size; ++i) {
+      const int tensor_index = interpreter_->inputs()[i];
+      TfLiteTensor* tensor = interpreter_->tensor(tensor_index);
+      float* tensor_buffer = tensor->data.f;
+      ASSERT_NE(tensor_buffer, nullptr);
+      for (int j = 0; j < width * height * channels; ++j) {
+        tensor_buffer[j] = i;
+      }
+      input_vec_->push_back(*tensor);
+      input_buffers_[i] = tensor_buffer;
+    }
+  }
+
+  void ValidateVectorOutput(std::vector<Packet>& output_packets,
+                            int expected_elements, int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+    const std::vector<TfLiteTensor>& output_vec =
+        output_packets[0].Get<std::vector<TfLiteTensor>>();
+    ASSERT_EQ(expected_elements, output_vec.size());
+
+    for (int i = 0; i < expected_elements; ++i) {
+      const int expected_value = input_begin_index + i;
+      const TfLiteTensor* result = &output_vec[i];
+      float* result_buffer = result->data.f;
+      ASSERT_NE(result_buffer, nullptr);
+      ASSERT_EQ(result_buffer, input_buffers_[input_begin_index + i]);
+      for (int j = 0; j < width * height * channels; ++j) {
+        ASSERT_EQ(expected_value, result_buffer[j]);
+      }
+    }
+  }
+
+  void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
+                                    int expected_elements,
+                                    std::vector<int>& input_begin_indices,
+                                    std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const std::vector<TfLiteTensor>& output_vec =
+        output_packets[0].Get<std::vector<TfLiteTensor>>();
+    ASSERT_EQ(expected_elements, output_vec.size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const int expected_value = i;
+        const TfLiteTensor* result = &output_vec[element_id];
+        float* result_buffer = result->data.f;
+        ASSERT_NE(result_buffer, nullptr);
+        ASSERT_EQ(result_buffer, input_buffers_[i]);
+        for (int j = 0; j < width * height * channels; ++j) {
+          ASSERT_EQ(expected_value, result_buffer[j]);
+        }
+        element_id++;
+      }
+    }
+  }
+
+  void ValidateElementOutput(std::vector<Packet>& output_packets,
+                             int input_begin_index) {
+    ASSERT_EQ(1, output_packets.size());
+
+    const TfLiteTensor& result = output_packets[0].Get<TfLiteTensor>();
+    float* result_buffer = result.data.f;
+    ASSERT_NE(result_buffer, nullptr);
+    ASSERT_EQ(result_buffer, input_buffers_[input_begin_index]);
+
+    const int expected_value = input_begin_index;
+    for (int j = 0; j < width * height * channels; ++j) {
+      ASSERT_EQ(expected_value, result_buffer[j]);
+    }
+  }
+
+  std::unique_ptr<Interpreter> interpreter_ = absl::make_unique<Interpreter>();
+  std::unique_ptr<std::vector<TfLiteTensor>> input_vec_ = nullptr;
+  std::vector<float*> input_buffers_;
+  std::unique_ptr<CalculatorRunner> runner_ = nullptr;
+};
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  PrepareTfLiteTensorVector(/*vector_size=*/5);
+  ASSERT_NE(input_vec_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/0);
+  ValidateVectorOutput(range_1_packets, /*expected_elements=*/3,
+                       /*input_begin_index=*/1);
+  ValidateVectorOutput(range_2_packets, /*expected_elements=*/1,
+                       /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 0 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because of an invalid range (begin == end).
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest,
+       InvalidCombineOutputsMultipleOutputsTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  PrepareTfLiteTensorVector(/*vector_size=*/5);
+  ASSERT_NE(input_vec_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+  std::vector<Packet> range_1_packets;
+  tool::AddVectorSink("range_1", &graph_config, &range_1_packets);
+  std::vector<Packet> range_2_packets;
+  tool::AddVectorSink("range_2", &graph_config, &range_2_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ValidateElementOutput(range_0_packets,
+                        /*input_begin_index=*/0);
+  ValidateElementOutput(range_1_packets,
+                        /*input_begin_index=*/2);
+  ValidateElementOutput(range_2_packets,
+                        /*input_begin_index=*/4);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  PrepareTfLiteTensorVector(/*vector_size=*/5);
+  ASSERT_NE(input_vec_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
+                               input_begin_indices, input_end_indices);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest,
+       ElementOnlyDisablesVectorOutputs) {
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              output_stream: "range_2"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 1 end: 4 }
+                  ranges: { begin: 4 end: 5 }
+                  element_only: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/stream_to_side_packet_calculator.cc

@@ -0,0 +1,48 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+
+// A calculator that takes a packet of an input stream and converts it to an
+// output side packet. This calculator only works under the assumption that the
+// input stream only has a single packet passing through.
+//
+// Example config:
+// node {
+//   calculator: "StreamToSidePacketCalculator"
+//   input_stream: "stream"
+//   output_side_packet: "side_packet"
+// }
+class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc) {
+    cc->Inputs().Index(0).SetAny();
+    cc->OutputSidePackets().Index(0).SetAny();
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Process(mediapipe::CalculatorContext* cc) override {
+    mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
+    cc->OutputSidePackets().Index(0).Set(
+        packet.At(mediapipe::Timestamp::Unset()));
+    return mediapipe::OkStatus();
+  }
+};
+REGISTER_CALCULATOR(StreamToSidePacketCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/stream_to_side_packet_calculator_test.cc

@@ -0,0 +1,67 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+
+using ::testing::Test;
+
+class StreamToSidePacketCalculatorTest : public Test {
+ protected:
+  StreamToSidePacketCalculatorTest() {
+    const char kConfig[] = R"(
+      calculator: "StreamToSidePacketCalculator"
+      input_stream: "stream"
+      output_side_packet: "side_packet"
+    )";
+    runner_ = absl::make_unique<CalculatorRunner>(kConfig);
+  }
+
+  std::unique_ptr<CalculatorRunner> runner_;
+};
+
+TEST_F(StreamToSidePacketCalculatorTest,
+       StreamToSidePacketCalculatorWithEmptyStreamFails) {
+  EXPECT_EQ(runner_->Run().code(), mediapipe::StatusCode::kUnavailable);
+}
+
+TEST_F(StreamToSidePacketCalculatorTest,
+       StreamToSidePacketCalculatorWithSinglePacketCreatesSidePacket) {
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(new std::string("test")).At(Timestamp(1)));
+  MP_ASSERT_OK(runner_->Run());
+  EXPECT_EQ(runner_->OutputSidePackets().Index(0).Get<std::string>(), "test");
+}
+
+TEST_F(StreamToSidePacketCalculatorTest,
+       StreamToSidePacketCalculatorWithMultiplePacketsFails) {
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(new std::string("test1")).At(Timestamp(1)));
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(new std::string("test2")).At(Timestamp(2)));
+  EXPECT_EQ(runner_->Run().code(), mediapipe::StatusCode::kAlreadyExists);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/string_to_int_calculator.cc

@@ -0,0 +1,79 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/types.h>
+
+#include <memory>
+#include <string>
+
+#include "absl/strings/numbers.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator that converts a std::string into an integer type, or fails if the
+// conversion is not possible.
+//
+// Example config:
+// node {
+//   calculator: "StringToIntCalculator"
+//   input_side_packet: "string"
+//   output_side_packet: "index"
+// }
+template <typename IntType>
+class StringToIntCalculatorTemplate : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets().Index(0).Set<std::string>();
+    cc->OutputSidePackets().Index(0).Set<IntType>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    IntType number;
+    if (!absl::SimpleAtoi(cc->InputSidePackets().Index(0).Get<std::string>(),
+                          &number)) {
+      return ::mediapipe::InvalidArgumentError(
+          "The std::string could not be parsed as an integer.");
+    }
+    cc->OutputSidePackets().Index(0).Set(MakePacket<IntType>(number));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+};
+
+using StringToIntCalculator = StringToIntCalculatorTemplate<int>;
+REGISTER_CALCULATOR(StringToIntCalculator);
+
+using StringToUintCalculator = StringToIntCalculatorTemplate<uint>;
+REGISTER_CALCULATOR(StringToUintCalculator);
+
+using StringToInt32Calculator = StringToIntCalculatorTemplate<int32>;
+REGISTER_CALCULATOR(StringToInt32Calculator);
+
+using StringToUint32Calculator = StringToIntCalculatorTemplate<uint32>;
+REGISTER_CALCULATOR(StringToUint32Calculator);
+
+using StringToInt64Calculator = StringToIntCalculatorTemplate<int64>;
+REGISTER_CALCULATOR(StringToInt64Calculator);
+
+using StringToUint64Calculator = StringToIntCalculatorTemplate<uint64>;
+REGISTER_CALCULATOR(StringToUint64Calculator);
+
+}  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
 exports_files(["LICENSE"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "opencv_image_encoder_calculator_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
@@ -46,6 +46,26 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "image_cropping_calculator_proto",
+    srcs = ["image_cropping_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+proto_library(
+    name = "bilateral_filter_calculator_proto",
+    srcs = ["bilateral_filter_calculator.proto"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 proto_library(
     name = "recolor_calculator_proto",
     srcs = ["recolor_calculator.proto"],
@@ -60,7 +80,7 @@ mediapipe_cc_proto_library(
     name = "opencv_image_encoder_calculator_cc_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
@@ -68,7 +88,7 @@ mediapipe_cc_proto_library(
     name = "mask_overlay_calculator_cc_proto",
     srcs = ["mask_overlay_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":mask_overlay_calculator_proto"],
 )
 
@@ -79,7 +99,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":scale_image_calculator_proto"],
 )
 
@@ -89,10 +109,30 @@ mediapipe_cc_proto_library(
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":set_alpha_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "image_cropping_calculator_cc_proto",
+    srcs = ["image_cropping_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":image_cropping_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "bilateral_filter_calculator_cc_proto",
+    srcs = ["bilateral_filter_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [":bilateral_filter_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "recolor_calculator_cc_proto",
     srcs = ["recolor_calculator.proto"],
@@ -100,7 +140,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/util:color_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":recolor_calculator_proto"],
 )
 
@@ -186,13 +226,43 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_quad_renderer",
+            "//mediapipe/gpu:shader_util",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "bilateral_filter_calculator",
+    srcs = ["bilateral_filter_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        ":bilateral_filter_calculator_cc_proto",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:vector",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gl_calculator_helper",
+            "//mediapipe/gpu:gl_simple_shaders",
+            "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -214,13 +284,26 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/gpu:scale_mode_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":image_transformation_calculator_proto"],
 )
 
 cc_library(
     name = "image_transformation_calculator",
     srcs = ["image_transformation_calculator.cc"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+        ],
+        "//conditions:default": [],
+    }),
+    linkopts = select({
+        "//mediapipe:apple": [
+            "-framework CoreVideo",
+            "-framework MetalKit",
+        ],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
     deps = [
         ":image_transformation_calculator_cc_proto",
@@ -233,13 +316,13 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -247,8 +330,24 @@ cc_library(
 cc_library(
     name = "image_cropping_calculator",
     srcs = ["image_cropping_calculator.cc"],
-    visibility = ["//visibility:public"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+        ],
+        "//conditions:default": [],
+    }),
+    linkopts = select({
+        "//mediapipe:apple": [
+            "-framework CoreVideo",
+            "-framework MetalKit",
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = [
+        "//visibility:public",
+    ],
     deps = [
+        ":image_cropping_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
@@ -257,7 +356,16 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-    ],
+        "//mediapipe/gpu:gpu_buffer",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gl_calculator_helper",
+            "//mediapipe/gpu:gl_simple_shaders",
+            "//mediapipe/gpu:gl_quad_renderer",
+            "//mediapipe/gpu:shader_util",
+        ],
+    }),
     alwayslink = 1,
 )
 
@@ -301,13 +409,13 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:color_cc_proto",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -357,6 +465,24 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "image_properties_calculator",
+    srcs = ["image_properties_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gpu_buffer",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_test(
     name = "opencv_encoded_image_to_image_frame_calculator_test",
     srcs = ["opencv_encoded_image_to_image_frame_calculator_test.cc"],

mediapipe/calculators/image/bilateral_filter_calculator.cc

@@ -0,0 +1,560 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/vector.h"
+
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gl_simple_shaders.h"
+#include "mediapipe/gpu/shader_util.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+
+namespace {
+constexpr char kInputFrameTag[] = "IMAGE";
+constexpr char kInputGuideTag[] = "GUIDE";
+constexpr char kOutputFrameTag[] = "IMAGE";
+
+constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
+constexpr char kInputGuideTagGpu[] = "GUIDE_GPU";
+constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
+
+enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+}  // namespace
+
+// A calculator for applying a bilateral filter to an image,
+// with an optional guide image (joint blateral).
+//
+// Inputs:
+//   One of the following two IMAGE tags:
+//   IMAGE: ImageFrame containing input image - Grayscale or RGB only.
+//   IMAGE_GPU: GpuBuffer containing input image - Grayscale, RGB or RGBA.
+//
+//   GUIDE (optional): ImageFrame guide image used to filter IMAGE. (N/A).
+//   GUIDE_GPU (optional): GpuBuffer guide image used to filter IMAGE_GPU.
+//
+// Output:
+//   One of the following two tags:
+//   IMAGE:     A filtered ImageFrame - Same as input.
+//   IMAGE_GPU:  A filtered GpuBuffer - RGBA
+//
+// Options:
+//   sigma_space: Pixel radius: use (sigma_space*2+1)x(sigma_space*2+1) window.
+//                This should be set based on output image pixel space.
+//   sigma_color: Color variance: normalized [0-1] color difference allowed.
+//
+// Notes:
+//   * When GUIDE is present, the output image is same size as GUIDE image;
+//     otherwise, the output image is same size as input image.
+//   * On GPU the kernel window is subsampled by approximately sqrt(sigma_space)
+//     i.e. the step size is ~sqrt(sigma_space),
+//     prioritizing performance > quality.
+//   * TODO: Add CPU path for joint filter.
+//
+class BilateralFilterCalculator : public CalculatorBase {
+ public:
+  BilateralFilterCalculator() = default;
+  ~BilateralFilterCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  // From Calculator.
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::Status RenderGpu(CalculatorContext* cc);
+  ::mediapipe::Status RenderCpu(CalculatorContext* cc);
+
+  ::mediapipe::Status GlSetup(CalculatorContext* cc);
+  void GlRender(CalculatorContext* cc);
+
+  mediapipe::BilateralFilterCalculatorOptions options_;
+  float sigma_color_ = -1.f;
+  float sigma_space_ = -1.f;
+
+  bool use_gpu_ = false;
+  bool gpu_initialized_ = false;
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  GLuint program_ = 0;
+  GLuint program_joint_ = 0;
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+};
+REGISTER_CALCULATOR(BilateralFilterCalculator);
+
+::mediapipe::Status BilateralFilterCalculator::GetContract(
+    CalculatorContract* cc) {
+  CHECK_GE(cc->Inputs().NumEntries(), 1);
+
+  if (cc->Inputs().HasTag(kInputFrameTag) &&
+      cc->Inputs().HasTag(kInputFrameTagGpu)) {
+    return ::mediapipe::InternalError("Cannot have multiple input images.");
+  }
+  if (cc->Inputs().HasTag(kInputFrameTagGpu) !=
+      cc->Outputs().HasTag(kOutputFrameTagGpu)) {
+    return ::mediapipe::InternalError("GPU output must have GPU input.");
+  }
+
+  bool use_gpu = false;
+
+  // Input image to filter.
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
+    cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  if (cc->Inputs().HasTag(kInputFrameTag)) {
+    cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
+  }
+
+  // Input guide image mask (optional)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  if (cc->Inputs().HasTag(kInputGuideTagGpu)) {
+    cc->Inputs().Tag(kInputGuideTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  if (cc->Inputs().HasTag(kInputGuideTag)) {
+    cc->Inputs().Tag(kInputGuideTag).Set<ImageFrame>();
+  }
+
+  // Output image.
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  if (cc->Outputs().HasTag(kOutputFrameTagGpu)) {
+    cc->Outputs().Tag(kOutputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  if (cc->Outputs().HasTag(kOutputFrameTag)) {
+    cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
+  }
+
+  if (use_gpu) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  options_ = cc->Options<mediapipe::BilateralFilterCalculatorOptions>();
+
+  if (cc->Inputs().HasTag(kInputFrameTagGpu) &&
+      cc->Outputs().HasTag(kOutputFrameTagGpu)) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    use_gpu_ = true;
+#else
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
+#endif
+  }
+
+  sigma_color_ = options_.sigma_color();
+  sigma_space_ = options_.sigma_space();
+  CHECK_GE(sigma_color_, 0.0);
+  CHECK_GE(sigma_space_, 0.0);
+  if (!use_gpu_) sigma_color_ *= 255.0;
+
+  if (use_gpu_) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BilateralFilterCalculator::Process(CalculatorContext* cc) {
+  if (use_gpu_) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(
+        gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
+          if (!gpu_initialized_) {
+            MP_RETURN_IF_ERROR(GlSetup(cc));
+            gpu_initialized_ = true;
+          }
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
+          return ::mediapipe::OkStatus();
+        }));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  } else {
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BilateralFilterCalculator::Close(CalculatorContext* cc) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  gpu_helper_.RunInGlContext([this] {
+    if (program_) glDeleteProgram(program_);
+    program_ = 0;
+    if (program_joint_) glDeleteProgram(program_joint_);
+    program_joint_ = 0;
+  });
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BilateralFilterCalculator::RenderCpu(
+    CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputFrameTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+
+  const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
+  auto input_mat = mediapipe::formats::MatView(&input_frame);
+
+  // Only 1 or 3 channel images supported by OpenCV.
+  if ((input_mat.channels() == 1 || input_mat.channels() == 3)) {
+    return ::mediapipe::InternalError(
+        "CPU filtering supports only 1 or 3 channel input images.");
+  }
+
+  auto output_frame = absl::make_unique<ImageFrame>(
+      input_frame.Format(), input_mat.cols, input_mat.rows);
+  const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTag) &&
+                               !cc->Inputs().Tag(kInputGuideTag).IsEmpty();
+
+  if (has_guide_image) {
+    // cv::jointBilateralFilter() is in contrib module 'ximgproc'.
+    return ::mediapipe::UnimplementedError(
+        "CPU joint filtering support is not implemented yet.");
+  } else {
+    auto output_mat = mediapipe::formats::MatView(output_frame.get());
+    // Prefer setting 'd = sigma_space * 2' to match GPU definition of radius.
+    cv::bilateralFilter(input_mat, output_mat, /*d=*/sigma_space_ * 2.0,
+                        sigma_color_, sigma_space_);
+  }
+
+  cc->Outputs()
+      .Tag(kOutputFrameTag)
+      .Add(output_frame.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BilateralFilterCalculator::RenderGpu(
+    CalculatorContext* cc) {
+  if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  const auto& input_frame =
+      cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
+  auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
+
+  mediapipe::GlTexture output_texture;
+  const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu) &&
+                               !cc->Inputs().Tag(kInputGuideTagGpu).IsEmpty();
+
+  // Setup textures and Update image in GPU shader.
+  if (has_guide_image) {
+    // joint bilateral filter
+    glUseProgram(program_joint_);
+    const auto& guide_image =
+        cc->Inputs().Tag(kInputGuideTagGpu).Get<mediapipe::GpuBuffer>();
+    auto guide_texture = gpu_helper_.CreateSourceTexture(guide_image);
+    glUniform2f(glGetUniformLocation(program_joint_, "texel_size_guide"),
+                1.0 / guide_image.width(), 1.0 / guide_image.height());
+    output_texture = gpu_helper_.CreateDestinationTexture(
+        guide_image.width(), guide_image.height(),
+        mediapipe::GpuBufferFormat::kBGRA32);
+    gpu_helper_.BindFramebuffer(output_texture);
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(GL_TEXTURE_2D, input_texture.name());
+    glActiveTexture(GL_TEXTURE2);
+    glBindTexture(GL_TEXTURE_2D, guide_texture.name());
+    GlRender(cc);
+    glActiveTexture(GL_TEXTURE2);
+    glBindTexture(GL_TEXTURE_2D, 0);
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(GL_TEXTURE_2D, 0);
+    guide_texture.Release();
+  } else {
+    // regular bilateral filter
+    glUseProgram(program_);
+    glUniform2f(glGetUniformLocation(program_, "texel_size"),
+                1.0 / input_frame.width(), 1.0 / input_frame.height());
+    output_texture = gpu_helper_.CreateDestinationTexture(
+        input_frame.width(), input_frame.height(),
+        mediapipe::GpuBufferFormat::kBGRA32);
+    gpu_helper_.BindFramebuffer(output_texture);
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(GL_TEXTURE_2D, input_texture.name());
+    GlRender(cc);
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(GL_TEXTURE_2D, 0);
+  }
+  glFlush();
+
+  // Send out image as GPU packet.
+  auto output_frame = output_texture.GetFrame<mediapipe::GpuBuffer>();
+  cc->Outputs()
+      .Tag(kOutputFrameTagGpu)
+      .Add(output_frame.release(), cc->InputTimestamp());
+
+  // Cleanup
+  input_texture.Release();
+  output_texture.Release();
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
+void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  static const GLfloat square_vertices[] = {
+      -1.0f, -1.0f,  // bottom left
+      1.0f,  -1.0f,  // bottom right
+      -1.0f, 1.0f,   // top left
+      1.0f,  1.0f,   // top right
+  };
+  static const GLfloat texture_vertices[] = {
+      0.0f, 0.0f,  // bottom left
+      1.0f, 0.0f,  // bottom right
+      0.0f, 1.0f,  // top left
+      1.0f, 1.0f,  // top right
+  };
+
+  // vertex storage
+  GLuint vbo[2];
+  glGenBuffers(2, vbo);
+  GLuint vao;
+  glGenVertexArrays(1, &vao);
+  glBindVertexArray(vao);
+
+  // vbo 0
+  glBindBuffer(GL_ARRAY_BUFFER, vbo[0]);
+  glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices,
+               GL_STATIC_DRAW);
+  glEnableVertexAttribArray(ATTRIB_VERTEX);
+  glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr);
+
+  // vbo 1
+  glBindBuffer(GL_ARRAY_BUFFER, vbo[1]);
+  glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices,
+               GL_STATIC_DRAW);
+  glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+  glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr);
+
+  // draw
+  glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+  // cleanup
+  glDisableVertexAttribArray(ATTRIB_VERTEX);
+  glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+  glBindVertexArray(0);
+  glDeleteVertexArrays(1, &vao);
+  glDeleteBuffers(2, vbo);
+
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+}
+
+::mediapipe::Status BilateralFilterCalculator::GlSetup(CalculatorContext* cc) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  const GLint attr_location[NUM_ATTRIBUTES] = {
+      ATTRIB_VERTEX,
+      ATTRIB_TEXTURE_POSITION,
+  };
+  const GLchar* attr_name[NUM_ATTRIBUTES] = {
+      "position",
+      "texture_coordinate",
+  };
+
+  // We bake our sigma values directly into the shader, so the GLSL compiler can
+  // optimize appropriately.
+  std::string sigma_options_string =
+      "const float sigma_space = " + std::to_string(sigma_space_) +
+      "; const float sigma_color = " + std::to_string(sigma_color_) + ";\n";
+
+  // Shader to do bilateral filtering on input image based on sigma space/color.
+  // Large kernel sizes are subsampled based on sqrt(sigma_space) window size,
+  // denoted as 'sparsity' below.
+  const std::string frag_src = GLES_VERSION_COMPAT
+                               R"(
+  #if __VERSION__ < 130
+    #define in varying
+  #endif  // __VERSION__ < 130
+
+  #ifdef GL_ES
+    #define fragColor gl_FragColor
+    precision highp float;
+  #else
+    #define lowp
+    #define mediump
+    #define highp
+    #define texture2D texture
+    out vec4 fragColor;
+  #endif  // defined(GL_ES)
+
+    in vec2 sample_coordinate;
+    uniform sampler2D input_frame;
+)" + sigma_options_string + R"(
+    uniform vec2 texel_size;
+
+    const float kSparsityFactor = 0.66;  // Higher is more sparse.
+    const float sparsity = max(1.0, sqrt(sigma_space) * kSparsityFactor);
+    const float step = sparsity;
+    const float radius = sigma_space;
+    const float offset = (step > 1.0) ? (step * 0.5) : (0.0);
+
+    float gaussian(float x, float sigma) {
+      float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
+      return exp((x * x) * coeff);
+    }
+
+    void main() {
+      vec2 center_uv = sample_coordinate;
+      vec3 center_val = texture2D(input_frame, center_uv).rgb;
+      vec3 new_val = vec3(0.0);
+
+      float space_weight = 0.0;
+      float color_weight = 0.0;
+      float total_weight = 0.0;
+
+      float sigma_texel = max(texel_size.x, texel_size.y) * sigma_space;
+      // Subsample kernel space.
+      for (float i = -radius+offset; i <= radius; i+=step) {
+        for (float j = -radius+offset; j <= radius; j+=step) {
+          vec2 shift = vec2(j, i) * texel_size;
+          vec2 uv = vec2(center_uv + shift);
+          vec3 val = texture2D(input_frame, uv).rgb;
+
+          space_weight = gaussian(distance(center_uv, uv), sigma_texel);
+          color_weight = gaussian(distance(center_val, val), sigma_color);
+          total_weight += space_weight * color_weight;
+
+          new_val += vec3(space_weight * color_weight) * val;
+        }
+      }
+      new_val /= vec3(total_weight);
+
+      fragColor = vec4(new_val, 1.0);
+    }
+  )";
+
+  // Create shader program and set parameters.
+  mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(),
+                              NUM_ATTRIBUTES, (const GLchar**)&attr_name[0],
+                              attr_location, &program_);
+  RET_CHECK(program_) << "Problem initializing the program.";
+  glUseProgram(program_);
+  glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
+
+  // Shader to do joint bilateral filtering on input image based on
+  // sigma space/color, and a Guide image.
+  // Large kernel sizes are subsampled based on sqrt(sigma_space) window size,
+  // denoted as 'sparsity' below.
+  const std::string joint_frag_src = GLES_VERSION_COMPAT
+                                     R"(
+  #if __VERSION__ < 130
+    #define in varying
+  #endif  // __VERSION__ < 130
+
+  #ifdef GL_ES
+    #define fragColor gl_FragColor
+    precision highp float;
+  #else
+    #define lowp
+    #define mediump
+    #define highp
+    #define texture2D texture
+    out vec4 fragColor;
+  #endif  // defined(GL_ES)
+
+    in vec2 sample_coordinate;
+    uniform sampler2D input_frame;
+    uniform sampler2D guide_frame;
+)" + sigma_options_string + R"(
+    uniform vec2 texel_size_guide; // size of guide and resulting filtered image
+
+    const float kSparsityFactor = 0.66;  // Higher is more sparse.
+    const float sparsity = max(1.0, sqrt(sigma_space) * kSparsityFactor);
+    const float step = sparsity;
+    const float radius = sigma_space;
+    const float offset = (step > 1.0) ? (step * 0.5) : (0.0);
+
+    float gaussian(float x, float sigma) {
+      float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6);
+      return exp((x * x) * coeff);
+    }
+
+    void main() {
+      vec2 center_uv = sample_coordinate;
+      vec3 center_val = texture2D(guide_frame, center_uv).rgb;
+      vec3 new_val = vec3(0.0);
+
+      float space_weight = 0.0;
+      float color_weight = 0.0;
+      float total_weight = 0.0;
+
+      float sigma_texel = max(texel_size_guide.x, texel_size_guide.y) * sigma_space;
+      // Subsample kernel space.
+      for (float i = -radius+offset; i <= radius; i+=step) {
+        for (float j = -radius+offset; j <= radius; j+=step) {
+          vec2 shift = vec2(j, i) * texel_size_guide;
+          vec2 uv = vec2(center_uv + shift);
+          vec3 guide_val = texture2D(guide_frame, uv).rgb;
+          vec3 out_val = texture2D(input_frame, uv).rgb;
+
+          space_weight = gaussian(distance(center_uv, uv), sigma_texel);
+          color_weight = gaussian(distance(center_val, guide_val), sigma_color);
+          total_weight += space_weight * color_weight;
+
+          new_val += vec3(space_weight * color_weight) * out_val;
+        }
+      }
+      new_val /= vec3(total_weight);
+
+      fragColor = vec4(new_val, 1.0);
+    }
+  )";
+
+  // Create shader program and set parameters.
+  mediapipe::GlhCreateProgram(
+      mediapipe::kBasicVertexShader, joint_frag_src.c_str(), NUM_ATTRIBUTES,
+      (const GLchar**)&attr_name[0], attr_location, &program_joint_);
+  RET_CHECK(program_joint_) << "Problem initializing the program.";
+  glUseProgram(program_joint_);
+  glUniform1i(glGetUniformLocation(program_joint_, "input_frame"), 1);
+  glUniform1i(glGetUniformLocation(program_joint_, "guide_frame"), 2);
+
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/bilateral_filter_calculator.proto

@@ -0,0 +1,20 @@
+// Options for BilateralFilterCalculator
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message BilateralFilterCalculatorOptions {
+  extend CalculatorOptions {
+    optional BilateralFilterCalculatorOptions ext = 255670209;
+  }
+
+  // Max variance in color allowed, based on normalized color values.
+  optional float sigma_color = 1;
+
+  // Window radius.
+  // Results in a '(sigma_space*2+1) x (sigma_space*2+1)' size kernel.
+  // This should be set based on output image pixel space.
+  optional float sigma_space = 2;
+}

mediapipe/calculators/image/color_convert_calculator.cc

@@ -1,3 +1,17 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cmath>
+
+#include "mediapipe/calculators/image/image_cropping_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -21,71 +24,202 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gl_simple_shaders.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/gpu/shader_util.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+namespace {
+enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
+}  // namespace
+
 namespace mediapipe {
 
+namespace {
+
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+constexpr char kRectTag[] = "RECT";
+constexpr char kNormRectTag[] = "NORM_RECT";
+constexpr char kHeightTag[] = "HEIGHT";
+constexpr char kImageTag[] = "IMAGE";
+constexpr char kImageGpuTag[] = "IMAGE_GPU";
+constexpr char kWidthTag[] = "WIDTH";
+
+}  // namespace
+
 // Crops the input texture to the given rectangle region. The rectangle can
 // be at arbitrary location on the image with rotation. If there's rotation, the
 // output texture will have the size of the input rectangle. The rotation should
 // be in radian, see rect.proto for detail.
-// Currently it only works for CPU.
 //
 // Input:
-//   IMAGE: ImageFrame representing the input image.
 //   One of the following two tags:
+//   IMAGE - ImageFrame representing the input image.
+//   IMAGE_GPU - GpuBuffer representing the input image.
+//   One of the following two tags (optional if WIDTH/HEIGHT is specified):
 //   RECT - A Rect proto specifying the width/height and location of the
 //          cropping rectangle.
 //   NORM_RECT - A NormalizedRect proto specifying the width/height and location
-//          of the cropping rectangle in normalized coordinates.
+//               of the cropping rectangle in normalized coordinates.
+//   Alternative tags to RECT (optional if RECT/NORM_RECT is specified):
+//   WIDTH - The desired width of the output cropped image,
+//           based on image center
+//   HEIGHT - The desired height of the output cropped image,
+//            based on image center
 //
 // Output:
-//   IMAGE - Cropped frames.
+//   One of the following two tags:
+//   IMAGE - Cropped ImageFrame
+//   IMAGE_GPU - Cropped GpuBuffer.
+//
+// Note: input_stream values take precedence over options defined in the graph.
+//
 class ImageCroppingCalculator : public CalculatorBase {
  public:
   ImageCroppingCalculator() = default;
   ~ImageCroppingCalculator() override = default;
 
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
 
  private:
   ::mediapipe::Status RenderCpu(CalculatorContext* cc);
   ::mediapipe::Status RenderGpu(CalculatorContext* cc);
+  ::mediapipe::Status InitGpu(CalculatorContext* cc);
+  void GlRender();
+  void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
+                           int* dst_width, int* dst_height);
 
-  // TODO: Merge with GlCroppingCalculator to have GPU support.
+  mediapipe::ImageCroppingCalculatorOptions options_;
-  bool use_gpu_{};
+
+  bool use_gpu_ = false;
+  // Output texture corners (4) after transoformation in normalized coordinates.
+  float transformed_points_[8];
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  bool gpu_initialized_ = false;
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  GLuint program_ = 0;
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(ImageCroppingCalculator);
 
 ::mediapipe::Status ImageCroppingCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("IMAGE"));
+  RET_CHECK(cc->Inputs().HasTag(kImageTag) ^ cc->Inputs().HasTag(kImageGpuTag));
-  RET_CHECK(cc->Outputs().HasTag("IMAGE"));
+  RET_CHECK(cc->Outputs().HasTag(kImageTag) ^
+            cc->Outputs().HasTag(kImageGpuTag));
 
-  cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  bool use_gpu = false;
 
-  if (cc->Inputs().HasTag("RECT")) {
+  if (cc->Inputs().HasTag(kImageTag)) {
-    cc->Inputs().Tag("RECT").Set<Rect>();
+    RET_CHECK(cc->Outputs().HasTag(kImageTag));
+    cc->Inputs().Tag(kImageTag).Set<ImageFrame>();
+    cc->Outputs().Tag(kImageTag).Set<ImageFrame>();
   }
-  if (cc->Inputs().HasTag("NORM_RECT")) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    cc->Inputs().Tag("NORM_RECT").Set<NormalizedRect>();
+  if (cc->Inputs().HasTag(kImageGpuTag)) {
+    RET_CHECK(cc->Outputs().HasTag(kImageGpuTag));
+    cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag));
+  if (cc->Inputs().HasTag(kRectTag)) {
+    cc->Inputs().Tag(kRectTag).Set<Rect>();
+  }
+  if (cc->Inputs().HasTag(kNormRectTag)) {
+    cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>();
+  }
+  if (cc->Inputs().HasTag(kWidthTag)) {
+    cc->Inputs().Tag(kWidthTag).Set<int>();
+  }
+  if (cc->Inputs().HasTag(kHeightTag)) {
+    cc->Inputs().Tag(kHeightTag).Set<int>();
   }
 
-  cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
+  if (use_gpu) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status ImageCroppingCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  if (cc->Inputs().HasTag(kImageGpuTag)) {
+    use_gpu_ = true;
+  }
+
+  options_ = cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
+
+  if (use_gpu_) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+#else
+    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::Process(CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kRectTag) && cc->Inputs().Tag(kRectTag).IsEmpty()) {
+    VLOG(1) << "RECT is empty for timestamp: " << cc->InputTimestamp();
+    return ::mediapipe::OkStatus();
+  }
+  if (cc->Inputs().HasTag(kNormRectTag) &&
+      cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
+    VLOG(1) << "NORM_RECT is empty for timestamp: " << cc->InputTimestamp();
+    return ::mediapipe::OkStatus();
+  }
   if (use_gpu_) {
-    RETURN_IF_ERROR(RenderGpu(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+    MP_RETURN_IF_ERROR(
+        gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
+          if (!gpu_initialized_) {
+            MP_RETURN_IF_ERROR(InitGpu(cc));
+            gpu_initialized_ = true;
+          }
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
+          return ::mediapipe::OkStatus();
+        }));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    RETURN_IF_ERROR(RenderCpu(cc));
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
   }
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status ImageCroppingCalculator::Close(CalculatorContext* cc) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  gpu_helper_.RunInGlContext([this] {
+    if (program_) glDeleteProgram(program_);
+    program_ = 0;
+  });
+  gpu_initialized_ = false;
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+  const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
   float rect_center_x = input_img.Width() / 2.0f;
@@ -93,58 +227,282 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   float rotation = 0.0f;
   int target_width = input_img.Width();
   int target_height = input_img.Height();
-  if (cc->Inputs().HasTag("RECT")) {
+  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag("RECT").Get<Rect>();
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
         rect.y_center() >= 0) {
-      rotation = rect.rotation();
       rect_center_x = rect.x_center();
       rect_center_y = rect.y_center();
       target_width = rect.width();
       target_height = rect.height();
+      rotation = rect.rotation();
     }
-  } else if (cc->Inputs().HasTag("NORM_RECT")) {
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag("NORM_RECT").Get<NormalizedRect>();
+    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
     if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
         rect.y_center() >= 0.0) {
-      rotation = rect.rotation();
       rect_center_x = std::round(rect.x_center() * input_img.Width());
       rect_center_y = std::round(rect.y_center() * input_img.Height());
       target_width = std::round(rect.width() * input_img.Width());
       target_height = std::round(rect.height() * input_img.Height());
+      rotation = rect.rotation();
     }
-  }
-
-  cv::Mat rotated_mat;
-  if (std::abs(rotation) > 1e-5) {
-    // TODO: Use open source common math library.
-    const float pi = 3.1415926f;
-    rotation = rotation * 180.0 / pi;
-
-    // First rotation the image.
-    cv::Point2f src_center(rect_center_x, rect_center_y);
-    cv::Mat rotation_mat = cv::getRotationMatrix2D(src_center, rotation, 1.0);
-    cv::warpAffine(input_mat, rotated_mat, rotation_mat, input_mat.size());
   } else {
-    input_mat.copyTo(rotated_mat);
+    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
+      target_width = cc->Inputs().Tag(kWidthTag).Get<int>();
+      target_height = cc->Inputs().Tag(kHeightTag).Get<int>();
+    } else if (options_.has_width() && options_.has_height()) {
+      target_width = options_.width();
+      target_height = options_.height();
+    }
+    rotation = options_.rotation();
   }
 
-  // Then crop the requested area.
+  const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
-  const cv::Rect cropping_rect(rect_center_x - target_width / 2,
+                                 cv::Size2f(target_width, target_height),
-                               rect_center_y - target_height / 2, target_width,
+                                 rotation * 180.f / M_PI);
-                               target_height);
+  cv::Mat src_points;
-  cv::Mat cropped_image = cv::Mat(rotated_mat, cropping_rect);
+  cv::boxPoints(min_rect, src_points);
+
+  float dst_corners[8] = {0,
+                          min_rect.size.height - 1,
+                          0,
+                          0,
+                          min_rect.size.width - 1,
+                          0,
+                          min_rect.size.width - 1,
+                          min_rect.size.height - 1};
+  cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners);
+  cv::Mat projection_matrix =
+      cv::getPerspectiveTransform(src_points, dst_points);
+  cv::Mat cropped_image;
+  cv::warpPerspective(input_mat, cropped_image, projection_matrix,
+                      cv::Size(min_rect.size.width, min_rect.size.height));
 
   std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
       input_img.Format(), cropped_image.cols, cropped_image.rows));
   cv::Mat output_mat = formats::MatView(output_frame.get());
   cropped_image.copyTo(output_mat);
-  cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kImageTag).Add(output_frame.release(),
+                                   cc->InputTimestamp());
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
-  return ::mediapipe::UnimplementedError("GPU support is not implemented yet.");
+  if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  const Packet& input_packet = cc->Inputs().Tag(kImageGpuTag).Value();
+  const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
+  auto src_tex = gpu_helper_.CreateSourceTexture(input_buffer);
+
+  int out_width, out_height;
+  GetOutputDimensions(cc, src_tex.width(), src_tex.height(), &out_width,
+                      &out_height);
+  auto dst_tex = gpu_helper_.CreateDestinationTexture(out_width, out_height);
+
+  // Run cropping shader on GPU.
+  {
+    gpu_helper_.BindFramebuffer(dst_tex);  // GL_TEXTURE0
+
+    glActiveTexture(GL_TEXTURE1);
+    glBindTexture(src_tex.target(), src_tex.name());
+
+    GlRender();
+
+    glActiveTexture(GL_TEXTURE2);
+    glBindTexture(GL_TEXTURE_2D, 0);
+    glFlush();
+  }
+
+  // Send result image in GPU packet.
+  auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
+  cc->Outputs().Tag(kImageGpuTag).Add(output.release(), cc->InputTimestamp());
+
+  // Cleanup
+  src_tex.Release();
+  dst_tex.Release();
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
+void ImageCroppingCalculator::GlRender() {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  static const GLfloat square_vertices[] = {
+      -1.0f, -1.0f,  // bottom left
+      1.0f,  -1.0f,  // bottom right
+      -1.0f, 1.0f,   // top left
+      1.0f,  1.0f,   // top right
+  };
+  const GLfloat* texture_vertices = &transformed_points_[0];
+
+  // program
+  glUseProgram(program_);
+
+  // vertex storage
+  GLuint vbo[2];
+  glGenBuffers(2, vbo);
+  GLuint vao;
+  glGenVertexArrays(1, &vao);
+  glBindVertexArray(vao);
+
+  // vbo 0
+  glBindBuffer(GL_ARRAY_BUFFER, vbo[0]);
+  glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices,
+               GL_STATIC_DRAW);
+  glEnableVertexAttribArray(ATTRIB_VERTEX);
+  glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr);
+
+  // vbo 1
+  glBindBuffer(GL_ARRAY_BUFFER, vbo[1]);
+  glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices,
+               GL_STATIC_DRAW);
+  glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+  glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr);
+
+  // draw
+  glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+  // cleanup
+  glDisableVertexAttribArray(ATTRIB_VERTEX);
+  glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+  glBindVertexArray(0);
+  glDeleteVertexArrays(1, &vao);
+  glDeleteBuffers(2, vbo);
+
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+}
+
+::mediapipe::Status ImageCroppingCalculator::InitGpu(CalculatorContext* cc) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+  const GLint attr_location[NUM_ATTRIBUTES] = {
+      ATTRIB_VERTEX,
+      ATTRIB_TEXTURE_POSITION,
+  };
+  const GLchar* attr_name[NUM_ATTRIBUTES] = {
+      "position",
+      "texture_coordinate",
+  };
+
+  // Simple pass-through shader.
+  const GLchar* frag_src = GLES_VERSION_COMPAT
+      R"(
+  #if __VERSION__ < 130
+    #define in varying
+  #endif  // __VERSION__ < 130
+
+  #ifdef GL_ES
+    #define fragColor gl_FragColor
+    precision highp float;
+  #else
+    #define lowp
+    #define mediump
+    #define highp
+    #define texture2D texture
+    out vec4 fragColor;
+  #endif  // defined(GL_ES)
+
+    in vec2 sample_coordinate;
+    uniform sampler2D input_frame;
+
+    void main() {
+      vec4 pix = texture2D(input_frame, sample_coordinate);
+      fragColor = pix;
+    }
+  )";
+
+  // Program
+  mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src,
+                              NUM_ATTRIBUTES, &attr_name[0], attr_location,
+                              &program_);
+  RET_CHECK(program_) << "Problem initializing the program.";
+
+  // Parameters
+  glUseProgram(program_);
+  glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+  return ::mediapipe::OkStatus();
+}
+
+// For GPU only.
+void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
+                                                  int src_width, int src_height,
+                                                  int* dst_width,
+                                                  int* dst_height) {
+  // Get the size of the cropping box.
+  int crop_width = src_width;
+  int crop_height = src_height;
+  // Get the center of cropping box. Default is the at the center.
+  int x_center = src_width / 2;
+  int y_center = src_height / 2;
+  // Get the rotation of the cropping box.
+  float rotation = 0.0f;
+  if (cc->Inputs().HasTag(kRectTag)) {
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
+    // Only use the rect if it is valid.
+    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
+        rect.y_center() >= 0) {
+      x_center = rect.x_center();
+      y_center = rect.y_center();
+      crop_width = rect.width();
+      crop_height = rect.height();
+      rotation = rect.rotation();
+    }
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
+    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
+    // Only use the rect if it is valid.
+    if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
+        rect.y_center() >= 0.0) {
+      x_center = std::round(rect.x_center() * src_width);
+      y_center = std::round(rect.y_center() * src_height);
+      crop_width = std::round(rect.width() * src_width);
+      crop_height = std::round(rect.height() * src_height);
+      rotation = rect.rotation();
+    }
+  } else {
+    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
+      crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
+      crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
+    } else if (options_.has_width() && options_.has_height()) {
+      crop_width = options_.width();
+      crop_height = options_.height();
+    }
+    rotation = options_.rotation();
+  }
+
+  const float half_width = crop_width / 2.0f;
+  const float half_height = crop_height / 2.0f;
+  const float corners[] = {-half_width, -half_height, half_width, -half_height,
+                           -half_width, half_height,  half_width, half_height};
+
+  for (int i = 0; i < 4; ++i) {
+    const float rotated_x = std::cos(rotation) * corners[i * 2] -
+                            std::sin(rotation) * corners[i * 2 + 1];
+    const float rotated_y = std::sin(rotation) * corners[i * 2] +
+                            std::cos(rotation) * corners[i * 2 + 1];
+
+    transformed_points_[i * 2] = ((rotated_x + x_center) / src_width);
+    transformed_points_[i * 2 + 1] = ((rotated_y + y_center) / src_height);
+  }
+
+  // Find the boundaries of the transformed rectangle.
+  float col_min = transformed_points_[0];
+  float col_max = transformed_points_[0];
+  float row_min = transformed_points_[1];
+  float row_max = transformed_points_[1];
+  for (int i = 1; i < 4; ++i) {
+    col_min = std::min(col_min, transformed_points_[i * 2]);
+    col_max = std::max(col_max, transformed_points_[i * 2]);
+    row_min = std::min(row_min, transformed_points_[i * 2 + 1]);
+    row_max = std::max(row_max, transformed_points_[i * 2 + 1]);
+  }
+
+  *dst_width = std::round((col_max - col_min) * src_width);
+  *dst_height = std::round((row_max - row_min) * src_height);
 }
 
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.proto

@@ -0,0 +1,33 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ImageCroppingCalculatorOptions {
+  extend CalculatorOptions {
+    optional ImageCroppingCalculatorOptions ext = 262466399;
+  }
+
+  // Output texture buffer dimensions. The values defined in the options will be
+  // overriden by the WIDTH and HEIGHT input streams if they exist.
+  optional int32 width = 1;
+  optional int32 height = 2;
+
+  // Rotation angle is counter-clockwise in radian.
+  optional float rotation = 3 [default = 0.0];
+}