Project import generated by Copybara.

GitOrigin-RevId: 1a0caa03bbf3673dbe772c8045b687c6b6821bcc

.bazelrc

@@ -14,6 +14,7 @@ build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
 # Temporarily set the incompatiblity flag for Bazel 0.27.0 and above
 build --incompatible_disable_deprecated_attr_params=false
+build --incompatible_depset_is_not_iterable=false
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos

Dockerfile

@@ -24,6 +24,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
         ca-certificates \
+        curl \
         git \
         wget \
         unzip \
@@ -35,7 +36,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         libopencv-video-dev \
         software-properties-common && \
     add-apt-repository -y ppa:openjdk-r/ppa && \
-    apt-get update && apt-get install -y openjdk-11-jdk && \
+    apt-get update && apt-get install -y openjdk-8-jdk && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

README.md

@@ -5,6 +5,20 @@
 
 ![Real-time Face Detection](mediapipe/docs/images/realtime_face_detection.gif)
 
+> "<em>MediaPipe has made it extremely easy to build our 3D person pose reconstruction demo app, facilitating accelerated neural network inference on device and synchronization of our result visualization with the video capture stream. Highly recommended!</em>" - George Papandreou, CTO, [Ariel AI](https://arielai.com)
+
+## ML Solutions in MediaPipe
+
+* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
+* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
+* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
+* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
+
+![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
+![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
+![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
+
 ## Installation
 Follow these [instructions](mediapipe/docs/install.md).
 
@@ -23,10 +37,15 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 *  [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General community discussion around MediaPipe
 
 ## Publications
+* [On-Device, Real-Time Hand Tracking with MediaPipe](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
-[Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
+* [ML Conference, Berlin 9-11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
+* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
+* [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
+* [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan
+* [Open sourced at CVPR 2019](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe) on June 17~20 in Long Beach, CA
 
 ## Alpha Disclaimer
 MediaPipe is currently in alpha for v0.6. We are still making breaking API changes and expect to get to stable API by v1.0.

WORKSPACE

@@ -10,7 +10,8 @@ http_archive(
     sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.23.0")
+versions.check(minimum_bazel_version = "0.24.1",
+               maximum_bazel_version = "0.29.1")
 
 # ABSL cpp library.
 http_archive(
@@ -25,6 +26,12 @@ http_archive(
     strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
 )
 
+http_archive(
+    name = "rules_cc",
+    strip_prefix = "rules_cc-master",
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
+)
+
 # GoogleTest/GoogleMock framework. Used by most unit-tests.
 http_archive(
      name = "com_google_googletest",
@@ -58,6 +65,12 @@ http_archive(
     sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
     strip_prefix = "glog-0.3.5",
     build_file = "@//third_party:glog.BUILD",
+    patches = [
+        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
 )
 
 # libyuv
@@ -114,7 +127,9 @@ http_archive(
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
 tf_workspace(tf_repo_name = "org_tensorflow")
 
-# Please run $ sudo apt-get install libopencv-dev
+# Please run
+# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+#                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
     build_file = "@//third_party:opencv_linux.BUILD",
@@ -127,7 +142,7 @@ new_local_repository(
     path = "/usr"
 )
 
-# Please run $ brew install opencv
+# Please run $ brew install opencv@3
 new_local_repository(
     name = "macos_opencv",
     build_file = "@//third_party:opencv_macos.BUILD",

mediapipe/calculators/audio/BUILD

@@ -67,6 +67,23 @@ mediapipe_cc_proto_library(
     deps = [":spectrogram_calculator_proto"],
 )
 
+proto_library(
+    name = "stabilized_log_calculator_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "stabilized_log_calculator_cc_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":stabilized_log_calculator_proto"],
+)
+
 proto_library(
     name = "time_series_framer_calculator_proto",
     srcs = ["time_series_framer_calculator.proto"],
@@ -156,6 +173,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "stabilized_log_calculator",
+    srcs = ["stabilized_log_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "spectrogram_calculator",
     srcs = ["spectrogram_calculator.cc"],
@@ -266,6 +299,24 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "stabilized_log_calculator_test",
+    srcs = ["stabilized_log_calculator_test.cc"],
+    deps = [
+        ":stabilized_log_calculator",
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
 cc_test(
     name = "time_series_framer_calculator_test",
     srcs = ["time_series_framer_calculator_test.cc"],

mediapipe/calculators/audio/audio_decoder_calculator.cc

@@ -61,10 +61,12 @@ class AudioDecoderCalculator : public CalculatorBase {
 ::mediapipe::Status AudioDecoderCalculator::GetContract(
     CalculatorContract* cc) {
   cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
-
+  if (cc->InputSidePackets().HasTag("OPTIONS")) {
+    cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
+  }
   cc->Outputs().Tag("AUDIO").Set<Matrix>();
   if (cc->Outputs().HasTag("AUDIO_HEADER")) {
-    cc->Outputs().Tag("AUDIO_HEADER").Set<mediapipe::TimeSeriesHeader>();
+    cc->Outputs().Tag("AUDIO_HEADER").SetNone();
   }
   return ::mediapipe::OkStatus();
 }
@@ -72,9 +74,11 @@ class AudioDecoderCalculator : public CalculatorBase {
 ::mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
   const std::string& input_file_path =
       cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
-  const auto& decoder_options = cc->Options<mediapipe::AudioDecoderOptions>();
+  const auto& decoder_options =
+      tool::RetrieveOptions(cc->Options<mediapipe::AudioDecoderOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
   decoder_ = absl::make_unique<AudioDecoder>();
-  RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options));
+  MP_RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options));
   std::unique_ptr<mediapipe::TimeSeriesHeader> header =
       absl::make_unique<mediapipe::TimeSeriesHeader>();
   if (decoder_->FillAudioHeader(decoder_options.audio_stream(0), header.get())

mediapipe/calculators/audio/audio_decoder_calculator_test.cc

@@ -39,11 +39,10 @@ TEST(AudioDecoderCalculatorTest, TestWAV) {
       file::JoinPath("./",
                      "/mediapipe/calculators/audio/"
                      "testdata/sine_wave_1k_44100_mono_2_sec_wav.audio"));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(runner.Outputs()
-      runner.Outputs()
+                   .Tag("AUDIO_HEADER")
-          .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
-          .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
   const mediapipe::TimeSeriesHeader& header =
       runner.Outputs()
           .Tag("AUDIO_HEADER")
@@ -71,11 +70,10 @@ TEST(AudioDecoderCalculatorTest, Test48KWAV) {
       file::JoinPath("./",
                      "/mediapipe/calculators/audio/"
                      "testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio"));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(runner.Outputs()
-      runner.Outputs()
+                   .Tag("AUDIO_HEADER")
-          .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
-          .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
   const mediapipe::TimeSeriesHeader& header =
       runner.Outputs()
           .Tag("AUDIO_HEADER")
@@ -103,11 +101,10 @@ TEST(AudioDecoderCalculatorTest, TestMP3) {
       file::JoinPath("./",
                      "/mediapipe/calculators/audio/"
                      "testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio"));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(runner.Outputs()
-      runner.Outputs()
+                   .Tag("AUDIO_HEADER")
-          .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
-          .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
   const mediapipe::TimeSeriesHeader& header =
       runner.Outputs()
           .Tag("AUDIO_HEADER")
@@ -135,11 +132,10 @@ TEST(AudioDecoderCalculatorTest, TestAAC) {
       file::JoinPath("./",
                      "/mediapipe/calculators/audio/"
                      "testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio"));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(runner.Outputs()
-      runner.Outputs()
+                   .Tag("AUDIO_HEADER")
-          .Tag("AUDIO_HEADER")
+                   .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
-          .header.ValidateAsType<mediapipe::TimeSeriesHeader>());
   const mediapipe::TimeSeriesHeader& header =
       runner.Outputs()
           .Tag("AUDIO_HEADER")

mediapipe/calculators/audio/basic_time_series_calculators.cc

@@ -51,11 +51,11 @@ static bool SafeMultiply(int x, int y, int* result) {
 
 ::mediapipe::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) {
   TimeSeriesHeader input_header;
-  RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
   auto output_header = new TimeSeriesHeader(input_header);
-  RETURN_IF_ERROR(MutateHeader(output_header));
+  MP_RETURN_IF_ERROR(MutateHeader(output_header));
   cc->Outputs().Index(0).SetHeader(Adopt(output_header));
   return ::mediapipe::OkStatus();
 }
@@ -63,11 +63,11 @@ static bool SafeMultiply(int x, int y, int* result) {
 ::mediapipe::Status BasicTimeSeriesCalculatorBase::Process(
     CalculatorContext* cc) {
   const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
-  RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
+  MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
       input, cc->Inputs().Index(0).Header().Get<TimeSeriesHeader>()));
 
   std::unique_ptr<Matrix> output(new Matrix(ProcessMatrix(input)));
-  RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
+  MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader(
       *output, cc->Outputs().Index(0).Header().Get<TimeSeriesHeader>()));
 
   cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());

mediapipe/calculators/audio/mfcc_mel_calculators.cc

@@ -90,8 +90,8 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
  private:
   // Takes header and options, and sets up state including calling
   // set_num_output_channels() on the base object.
-  virtual ::mediapipe::Status ConfigureTransform(
+  virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-      const TimeSeriesHeader& header, const CalculatorOptions& options) = 0;
+                                                 CalculatorContext* cc) = 0;
 
   // Takes a vector<double> corresponding to an input frame, and
   // perform the specific transformation to produce an output frame.
@@ -105,10 +105,10 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
 ::mediapipe::Status FramewiseTransformCalculatorBase::Open(
     CalculatorContext* cc) {
   TimeSeriesHeader input_header;
-  RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
-  ::mediapipe::Status status = ConfigureTransform(input_header, cc->Options());
+  ::mediapipe::Status status = ConfigureTransform(input_header, cc);
 
   auto output_header = new TimeSeriesHeader(input_header);
   output_header->set_num_channels(num_output_channels_);
@@ -175,11 +175,9 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
   }
 
  private:
-  ::mediapipe::Status ConfigureTransform(
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-      const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
-      const CalculatorOptions& options) override {
+    MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
-    MfccCalculatorOptions mfcc_options;
-    time_series_util::FillOptionsExtensionOrDie(options, &mfcc_options);
     mfcc_.reset(new audio_dsp::Mfcc());
     int input_length = header.num_channels();
     // Set up the parameters to the Mfcc object.
@@ -235,11 +233,10 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
   }
 
  private:
-  ::mediapipe::Status ConfigureTransform(
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
-      const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
-      const CalculatorOptions& options) override {
+    MelSpectrumCalculatorOptions mel_spectrum_options =
-    MelSpectrumCalculatorOptions mel_spectrum_options;
+        cc->Options<MelSpectrumCalculatorOptions>();
-    time_series_util::FillOptionsExtensionOrDie(options, &mel_spectrum_options);
     mel_filterbank_.reset(new audio_dsp::MelFilterbank());
     int input_length = header.num_channels();
     set_num_output_channels(mel_spectrum_options.channel_count());

mediapipe/calculators/audio/mfcc_mel_calculators_test.cc

@@ -112,7 +112,7 @@ TEST_F(MfccCalculatorTest, AudioSampleRateFromInputHeader) {
   SetupGraphAndHeader();
   SetupRandomInputPackets();
 
-  MEDIAPIPE_EXPECT_OK(Run());
+  MP_EXPECT_OK(Run());
 
   CheckResults(options_.mfcc_count());
 }
@@ -134,7 +134,7 @@ TEST_F(MelSpectrumCalculatorTest, AudioSampleRateFromInputHeader) {
   SetupGraphAndHeader();
   SetupRandomInputPackets();
 
-  MEDIAPIPE_EXPECT_OK(Run());
+  MP_EXPECT_OK(Run());
 
   CheckResults(options_.channel_count());
 }

mediapipe/calculators/audio/rational_factor_resample_calculator.cc

@@ -64,8 +64,8 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
 
 ::mediapipe::Status RationalFactorResampleCalculator::Open(
     CalculatorContext* cc) {
-  RationalFactorResampleCalculatorOptions resample_options;
+  RationalFactorResampleCalculatorOptions resample_options =
-  time_series_util::FillOptionsExtensionOrDie(cc->Options(), &resample_options);
+      cc->Options<RationalFactorResampleCalculatorOptions>();
 
   if (!resample_options.has_target_sample_rate()) {
     return tool::StatusInvalid(
@@ -74,7 +74,7 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
   target_sample_rate_ = resample_options.target_sample_rate();
 
   TimeSeriesHeader input_header;
-  RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
   source_sample_rate_ = input_header.sample_rate();

mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc

@@ -209,25 +209,25 @@ class RationalFactorResampleCalculatorTest
 
 TEST_F(RationalFactorResampleCalculatorTest, Upsample) {
   const double kUpsampleRate = input_sample_rate_ * 1.9;
-  MEDIAPIPE_ASSERT_OK(Run(kUpsampleRate));
+  MP_ASSERT_OK(Run(kUpsampleRate));
   CheckOutput(kUpsampleRate);
 }
 
 TEST_F(RationalFactorResampleCalculatorTest, Downsample) {
   const double kDownsampleRate = input_sample_rate_ / 1.9;
-  MEDIAPIPE_ASSERT_OK(Run(kDownsampleRate));
+  MP_ASSERT_OK(Run(kDownsampleRate));
   CheckOutput(kDownsampleRate);
 }
 
 TEST_F(RationalFactorResampleCalculatorTest, UsesRationalFactorResampler) {
   const double kUpsampleRate = input_sample_rate_ * 2;
-  MEDIAPIPE_ASSERT_OK(Run(kUpsampleRate));
+  MP_ASSERT_OK(Run(kUpsampleRate));
   CheckOutput(kUpsampleRate);
 }
 
 TEST_F(RationalFactorResampleCalculatorTest, PassthroughIfSampleRateUnchanged) {
   const double kUpsampleRate = input_sample_rate_;
-  MEDIAPIPE_ASSERT_OK(Run(kUpsampleRate));
+  MP_ASSERT_OK(Run(kUpsampleRate));
   CheckOutputUnchanged();
 }
 
@@ -239,7 +239,7 @@ TEST_F(RationalFactorResampleCalculatorTest, DoesNotDieOnEmptyInput) {
   options_.set_target_sample_rate(input_sample_rate_);
   InitializeGraph();
   FillInputHeader();
-  MEDIAPIPE_ASSERT_OK(RunGraph());
+  MP_ASSERT_OK(RunGraph());
   EXPECT_TRUE(output().packets.empty());
 }
 

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -71,10 +71,8 @@ class SpectrogramCalculator : public CalculatorBase {
         // Input stream with TimeSeriesHeader.
     );
 
-    SpectrogramCalculatorOptions spectrogram_options;
+    SpectrogramCalculatorOptions spectrogram_options =
-    time_series_util::FillOptionsExtensionOrDie(cc->Options(),
+        cc->Options<SpectrogramCalculatorOptions>();
-                                                &spectrogram_options);
-
     if (!spectrogram_options.allow_multichannel_input()) {
       if (spectrogram_options.output_type() ==
           SpectrogramCalculatorOptions::COMPLEX) {
@@ -172,9 +170,8 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
 ::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
-  SpectrogramCalculatorOptions spectrogram_options;
+  SpectrogramCalculatorOptions spectrogram_options =
-  time_series_util::FillOptionsExtensionOrDie(cc->Options(),
+      cc->Options<SpectrogramCalculatorOptions>();
-                                              &spectrogram_options);
 
   if (spectrogram_options.frame_duration_seconds() <= 0.0) {
     ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
@@ -197,7 +194,7 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
   }
 
   TimeSeriesHeader input_header;
-  RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
   input_sample_rate_ = input_header.sample_rate();
@@ -223,6 +220,10 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   std::vector<double> window;
   switch (spectrogram_options.window_type()) {
+    case SpectrogramCalculatorOptions::COSINE:
+      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                   &window);
+      break;
     case SpectrogramCalculatorOptions::HANN:
       audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
                                                  &window);

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -58,6 +58,7 @@ message SpectrogramCalculatorOptions {
   enum WindowType {
     HANN = 0;
     HAMMING = 1;
+    COSINE = 2;
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/audio/spectrogram_calculator_test.cc

@@ -303,7 +303,7 @@ TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationNoOverlap) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -324,7 +324,7 @@ TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationSomeOverlap) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -344,7 +344,7 @@ TEST_F(SpectrogramCalculatorTest, NonintegerFrameDurationAndOverlap) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -365,7 +365,7 @@ TEST_F(SpectrogramCalculatorTest, ShortInitialPacketNoOverlap) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -382,7 +382,7 @@ TEST_F(SpectrogramCalculatorTest, TrailingSamplesNoPad) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -399,7 +399,7 @@ TEST_F(SpectrogramCalculatorTest, NoTrailingSamplesWithPad) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -418,7 +418,7 @@ TEST_F(SpectrogramCalculatorTest, TrailingSamplesWithPad) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -435,7 +435,7 @@ TEST_F(SpectrogramCalculatorTest, VeryShortInputWillPad) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -452,7 +452,7 @@ TEST_F(SpectrogramCalculatorTest, VeryShortInputZeroOutputFramesIfNoPad) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -468,7 +468,7 @@ TEST_F(SpectrogramCalculatorTest, DCSignalIsPeakBin) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   const float dc_frequency_hz = 0.0;
@@ -486,7 +486,7 @@ TEST_F(SpectrogramCalculatorTest, A440ToneIsPeakBin) {
   const float tone_frequency_hz = 440.0;
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   int num_output_frames = output().packets[0].Get<Matrix>().cols();
@@ -507,7 +507,7 @@ TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRight) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
@@ -525,7 +525,7 @@ TEST_F(SpectrogramCalculatorTest, DefaultOutputIsSquaredMagnitude) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
@@ -543,7 +543,7 @@ TEST_F(SpectrogramCalculatorTest, LinearMagnitudeOutputLooksRight) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
@@ -561,7 +561,7 @@ TEST_F(SpectrogramCalculatorTest, DbMagnitudeOutputLooksRight) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(output().packets[0].Get<Matrix>()(0, 0),
@@ -581,7 +581,7 @@ TEST_F(SpectrogramCalculatorTest, OutputScalingLooksRight) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(
@@ -600,7 +600,7 @@ TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRight) {
   // Setup packets with DC input (non-zero constant value).
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_FLOAT_EQ(std::norm(output().packets[0].Get<Eigen::MatrixXcf>()(0, 0)),
@@ -623,7 +623,7 @@ TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRightForImpulses) {
   // Make two impulse packets offset one sample from each other
   SetupImpulseInputPackets(input_packet_sizes, input_packet_impulse_offsets);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   const int num_buckets =
@@ -671,7 +671,7 @@ TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRightForNonDC) {
   const float tone_frequency_hz = target_bin * (input_sample_rate_ / fft_size);
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   // For a non-DC bin, the magnitude will be split between positive and
@@ -696,7 +696,7 @@ TEST_F(SpectrogramCalculatorTest, ZeroOutputsForZeroInputsWithPaddingEnabled) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes);
@@ -713,7 +713,7 @@ TEST_F(SpectrogramCalculatorTest, NumChannelsIsRight) {
   FillInputHeader();
   const float tone_frequency_hz = 440.0;
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   EXPECT_EQ(output().packets[0].Get<std::vector<Matrix>>().size(),
@@ -732,7 +732,7 @@ TEST_F(SpectrogramCalculatorTest, NumSamplesAndPacketRateAreCleared) {
   FillInputHeader();
   SetupConstantInputPackets(input_packet_sizes);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   const TimeSeriesHeader& output_header =
       output().header.Get<TimeSeriesHeader>();
@@ -751,7 +751,7 @@ TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramSizesAreRight) {
   FillInputHeader();
   const float tone_frequency_hz = 440.0;
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
@@ -776,7 +776,7 @@ TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramValuesAreRight) {
   const float tone_frequency_hz = 440.0;
   SetupMultichannelInputPackets(input_packet_sizes, tone_frequency_hz);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
@@ -805,7 +805,7 @@ TEST_F(SpectrogramCalculatorTest, MultichannelHandlesShortInitialPacket) {
   FillInputHeader();
   const float tone_frequency_hz = 440.0;
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   auto spectrograms = output().packets[0].Get<std::vector<Matrix>>();
@@ -833,7 +833,7 @@ TEST_F(SpectrogramCalculatorTest,
   FillInputHeader();
   const float tone_frequency_hz = 440.0;
   SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
 
   CheckOutputHeadersAndTimestamps();
   auto spectrograms = output().packets[0].Get<std::vector<Eigen::MatrixXcf>>();

mediapipe/calculators/audio/stabilized_log_calculator.cc

@@ -0,0 +1,99 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines StabilizedLogCalculator.
+
+#include <cmath>
+#include <memory>
+#include <string>
+
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "StabilizedLogCalculator"
+//   input_stream: "input_time_series"
+//   output_stream: "stabilized_log_time_series"
+//   options {
+//     [mediapipe.StabilizedLogCalculatorOptions.ext] {
+//       stabilizer: .00001
+//       check_nonnegativity: true
+//     }
+//   }
+// }
+class StabilizedLogCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input stream with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Output stabilized log stream with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    StabilizedLogCalculatorOptions stabilized_log_calculator_options =
+        cc->Options<StabilizedLogCalculatorOptions>();
+
+    stabilizer_ = stabilized_log_calculator_options.stabilizer();
+    output_scale_ = stabilized_log_calculator_options.output_scale();
+    check_nonnegativity_ =
+        stabilized_log_calculator_options.check_nonnegativity();
+    CHECK_GE(stabilizer_, 0.0)
+        << "stabilizer must be >= 0.0, received a value of " << stabilizer_;
+
+    // If the input packets have a header, propagate the header to the output.
+    if (!cc->Inputs().Index(0).Header().IsEmpty()) {
+      TimeSeriesHeader input_header;
+      MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+          cc->Inputs().Index(0).Header(), &input_header));
+      cc->Outputs().Index(0).SetHeader(
+          Adopt(new TimeSeriesHeader(input_header)));
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
+    if (input_matrix.array().isNaN().any()) {
+      return ::mediapipe::InvalidArgumentError("NaN input to log operation.");
+    }
+    if (check_nonnegativity_) {
+      if (input_matrix.minCoeff() < 0.0) {
+        return ::mediapipe::OutOfRangeError("Negative input to log operation.");
+      }
+    }
+    std::unique_ptr<Matrix> output_frame(new Matrix(
+        output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
+    cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float stabilizer_;
+  bool check_nonnegativity_;
+  double output_scale_;
+};
+REGISTER_CALCULATOR(StabilizedLogCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/stabilized_log_calculator.proto

@@ -0,0 +1,37 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message StabilizedLogCalculatorOptions {
+  extend CalculatorOptions {
+    optional StabilizedLogCalculatorOptions ext = 101978339;
+  }
+
+  // The calculator computes log(x + stabilizer). stabilizer must be >=
+  // 0, with 0 indicating a lack of stabilization.
+  optional float stabilizer = 1 [default = .00001];
+
+  // If true, CHECK that all input values in are >= 0. If false, the
+  // code will take the log of the potentially negative input values
+  // plus the stabilizer.
+  optional bool check_nonnegativity = 2 [default = true];
+
+  // Support a fixed multiplicative scaling of the output.
+  optional double output_scale = 3 [default = 1.0];
+}

mediapipe/calculators/audio/stabilized_log_calculator_test.cc

@@ -0,0 +1,141 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <cmath>
+
+#include "Eigen/Core"
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+const float kStabilizer = 0.1;
+const int kNumChannels = 3;
+const int kNumSamples = 10;
+
+class StabilizedLogCalculatorTest
+    : public TimeSeriesCalculatorTest<StabilizedLogCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "StabilizedLogCalculator";
+    options_.set_stabilizer(kStabilizer);
+
+    input_sample_rate_ = 8000.0;
+    num_input_channels_ = kNumChannels;
+    num_input_samples_ = kNumSamples;
+  }
+
+  void RunGraphNoReturn() { MP_ASSERT_OK(RunGraph()); }
+};
+
+TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
+  const int kNumPackets = 5;
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        (input_data_matrices[output_packet].array() + kStabilizer).log(),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
+  const int kNumPackets = 5;
+  double output_scale = 2.5;
+  options_.set_output_scale(output_scale);
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        output_scale *
+            ((input_data_matrices[output_packet].array() + kStabilizer).log()),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)),
+                    0 /* timestamp */);
+  MP_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  ExpectApproximatelyEqual(
+      Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(),
+      runner_->Outputs().Index(0).packets[0].Get<Matrix>());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NanValuesReturnError) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, std::nanf(""))),
+      0 /* timestamp */);
+  ASSERT_FALSE(RunGraph().ok());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesReturnError) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  ASSERT_FALSE(RunGraph().ok());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {
+  options_.set_check_nonnegativity(false);
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  MP_ASSERT_OK(RunGraph());
+  // Results are undefined.
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -56,6 +56,14 @@ namespace mediapipe {
 // If pad_final_packet is true, all input samples will be emitted and the final
 // packet will be zero padded as necessary.  If pad_final_packet is false, some
 // samples may be dropped at the end of the stream.
+//
+// If use_local_timestamp is true, the output packet's timestamp is based on the
+// last sample of the packet. The timestamp of this sample is inferred by
+// input_packet_timesamp + local_sample_index / sampling_rate_. If false, the
+// output packet's timestamp is based on the cumulative timestamping, which is
+// done by adopting the timestamp of the first sample of the packet and this
+// sample's timestamp is inferred by initial_input_timestamp_ +
+// cumulative_completed_samples / sample_rate_.
 class TimeSeriesFramerCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -86,11 +94,26 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   void FrameOutput(CalculatorContext* cc);
 
   Timestamp CurrentOutputTimestamp() {
+    if (use_local_timestamp_) {
+      return current_timestamp_;
+    }
+    return CumulativeOutputTimestamp();
+  }
+
+  Timestamp CumulativeOutputTimestamp() {
     return initial_input_timestamp_ +
            round(cumulative_completed_samples_ / sample_rate_ *
                  Timestamp::kTimestampUnitsPerSecond);
   }
 
+  // Returns the timestamp of a sample on a base, which is usually the time
+  // stamp of a packet.
+  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
+                                   int64 number_of_samples) {
+    return timestamp_base + round(number_of_samples / sample_rate_ *
+                                  Timestamp::kTimestampUnitsPerSecond);
+  }
+
   // The number of input samples to advance after the current output frame is
   // emitted.
   int next_frame_step_samples() const {
@@ -118,14 +141,18 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   // any overlap).
   int64 cumulative_completed_samples_;
   Timestamp initial_input_timestamp_;
+  // The current timestamp is updated along with the incoming packets.
+  Timestamp current_timestamp_;
   int num_channels_;
 
   // Each entry in this deque consists of a single sample, i.e. a
-  // single column vector.
+  // single column vector, and its timestamp.
-  std::deque<Matrix> sample_buffer_;
+  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
 
   bool use_window_;
   Matrix window_;
+
+  bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
 
@@ -133,7 +160,8 @@ void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
   const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
 
   for (int i = 0; i < input_frame.cols(); ++i) {
-    sample_buffer_.emplace_back(input_frame.col(i));
+    sample_buffer_.emplace_back(std::make_pair(
+        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
   }
 
   cumulative_input_samples_ += input_frame.cols();
@@ -151,14 +179,16 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
         new Matrix(num_channels_, frame_duration_samples_));
     for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
          ++i) {
-      output_frame->col(i) = sample_buffer_.front();
+      output_frame->col(i) = sample_buffer_.front().first;
+      current_timestamp_ = sample_buffer_.front().second;
       sample_buffer_.pop_front();
     }
     const int frame_overlap_samples =
         frame_duration_samples_ - frame_step_samples;
     if (frame_overlap_samples > 0) {
       for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i];
+        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
+        current_timestamp_ = sample_buffer_[i].second;
       }
     } else {
       samples_still_to_drop_ = -frame_overlap_samples;
@@ -178,6 +208,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
 ::mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
   if (initial_input_timestamp_ == Timestamp::Unstarted()) {
     initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
   }
 
   EnqueueInput(cc);
@@ -195,7 +226,8 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
     std::unique_ptr<Matrix> output_frame(new Matrix);
     output_frame->setZero(num_channels_, frame_duration_samples_);
     for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i];
+      output_frame->col(i) = sample_buffer_[i].first;
+      current_timestamp_ = sample_buffer_[i].second;
     }
 
     cc->Outputs().Index(0).Add(output_frame.release(),
@@ -206,8 +238,8 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
 }
 
 ::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
-  TimeSeriesFramerCalculatorOptions framer_options;
+  TimeSeriesFramerCalculatorOptions framer_options =
-  time_series_util::FillOptionsExtensionOrDie(cc->Options(), &framer_options);
+      cc->Options<TimeSeriesFramerCalculatorOptions>();
 
   RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0)
       << "Invalid or missing frame_duration_seconds. "
@@ -219,7 +251,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
       << framer_options.frame_overlap_seconds();
 
   TimeSeriesHeader input_header;
-  RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+  MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
   sample_rate_ = input_header.sample_rate();
@@ -258,6 +290,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
   cumulative_output_frames_ = 0;
   samples_still_to_drop_ = 0;
   initial_input_timestamp_ = Timestamp::Unstarted();
+  current_timestamp_ = Timestamp::Unstarted();
 
   std::vector<double> window_vector;
   use_window_ = false;
@@ -282,6 +315,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
                                           frame_duration_samples_)
                   .cast<float>();
   }
+  use_local_timestamp_ = framer_options.use_local_timestamp();
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/audio/time_series_framer_calculator.proto

@@ -62,4 +62,11 @@ message TimeSeriesFramerCalculatorOptions {
     HANN = 2;
   }
   optional WindowFunction window_function = 4 [default = NONE];
+
+  // If use_local_timestamp is true, the output packet's timestamp is based on
+  // the last sample of the packet and it's inferred from the latest input
+  // packet's timestamp.  If false, the output packet's timestamp is based on
+  // the cumulative timestamping, which is inferred from the intial input
+  // timestamp and the cumulative number of samples.
+  optional bool use_local_timestamp = 6 [default = false];
 }

mediapipe/calculators/audio/time_series_framer_calculator_test.cc

@@ -35,6 +35,8 @@ namespace mediapipe {
 namespace {
 
 const int kInitialTimestampOffsetMicroseconds = 4;
+const int kGapBetweenPacketsInSeconds = 1;
+const int kUniversalInputPacketSize = 50;
 
 class TimeSeriesFramerCalculatorTest
     : public TimeSeriesCalculatorTest<TimeSeriesFramerCalculatorOptions> {
@@ -226,7 +228,7 @@ class TimeSeriesFramerCalculatorTest
 
 TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationNoOverlap) {
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
@@ -234,7 +236,7 @@ TEST_F(TimeSeriesFramerCalculatorTest,
        IntegerSampleDurationNoOverlapHammingWindow) {
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
@@ -242,14 +244,14 @@ TEST_F(TimeSeriesFramerCalculatorTest,
        IntegerSampleDurationNoOverlapHannWindow) {
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
 TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationAndOverlap) {
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_frame_overlap_seconds(40.0 / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
@@ -257,7 +259,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, NonintegerSampleDurationAndOverlap) {
   options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
   options_.set_frame_overlap_seconds(38.4 / input_sample_rate_);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
@@ -267,7 +269,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFrames) {
   // the 1100 input samples.
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_frame_overlap_seconds(-10.0 / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 10);
   CheckOutput();
 }
@@ -277,7 +279,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFramesLessSkip) {
   // the 1100 input samples.
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 6);
   CheckOutput();
 }
@@ -287,7 +289,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapWithPadding) {
   // on the sixth and last frame given 1100 sample input.
   options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
   options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 6);
   CheckOutput();
 }
@@ -297,7 +299,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, FixedFrameOverlap) {
   // results in ceil((1100 - 30) / 11) + 1 = 99 packets.
   options_.set_frame_duration_seconds(30 / input_sample_rate_);
   options_.set_frame_overlap_seconds((30.0 - 11.4) / input_sample_rate_);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 99);
   CheckOutput();
 }
@@ -308,7 +310,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameOverlap) {
   options_.set_frame_duration_seconds(30 / input_sample_rate_);
   options_.set_frame_overlap_seconds((30 - 11.4) / input_sample_rate_);
   options_.set_emulate_fractional_frame_overlap(true);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 95);
   CheckOutput();
 }
@@ -319,7 +321,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameSkip) {
   options_.set_frame_duration_seconds(30 / input_sample_rate_);
   options_.set_frame_overlap_seconds((30 - 41.4) / input_sample_rate_);
   options_.set_emulate_fractional_frame_overlap(true);
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   EXPECT_EQ(output().packets.size(), 27);
   CheckOutput();
 }
@@ -328,7 +330,7 @@ TEST_F(TimeSeriesFramerCalculatorTest, NoFinalPacketPadding) {
   options_.set_frame_duration_seconds(98.5 / input_sample_rate_);
   options_.set_pad_final_packet(false);
 
-  MEDIAPIPE_ASSERT_OK(Run());
+  MP_ASSERT_OK(Run());
   CheckOutput();
 }
 
@@ -369,7 +371,7 @@ class TimeSeriesFramerCalculatorWindowingSanityTest
     FillInputHeader();
     AppendInputPacket(new Matrix(Matrix::Ones(1, FrameDurationSamples())),
                       kInitialTimestampOffsetMicroseconds);
-    MEDIAPIPE_ASSERT_OK(RunGraph());
+    MP_ASSERT_OK(RunGraph());
     ASSERT_EQ(1, output().packets.size());
     ASSERT_NEAR(expected_average * FrameDurationSamples(),
                 output().packets[0].Get<Matrix>().sum(), 1e-5);
@@ -391,5 +393,93 @@ TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, HannWindowSanityCheck) {
   RunAndTestSinglePacketAverage(0.5f);
 }
 
-}  // anonymous namespace
+// A simple test class that checks the local packet time stamp. This class
+// generate a series of packets with and without gaps between packets and tests
+// the behavior with cumulative timestamping and local packet timestamping.
+class TimeSeriesFramerCalculatorTimestampingTest
+    : public TimeSeriesFramerCalculatorTest {
+ protected:
+  // Creates test input and saves a reference copy.
+  void InitializeInputForTimeStampingTest() {
+    concatenated_input_samples_.resize(0, num_input_channels_);
+    num_input_samples_ = 0;
+    for (int i = 0; i < 10; ++i) {
+      // This range of packet sizes was chosen such that some input
+      // packets will be smaller than the output packet size and other
+      // input packets will be larger.
+      int packet_size = kUniversalInputPacketSize;
+      double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+                                 num_input_samples_ / input_sample_rate_;
+      if (options_.use_local_timestamp()) {
+        timestamp_seconds += kGapBetweenPacketsInSeconds * i;
+      }
+
+      Matrix* data_frame =
+          NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
+
+      AppendInputPacket(data_frame, round(timestamp_seconds *
+                                          Timestamp::kTimestampUnitsPerSecond));
+      num_input_samples_ += packet_size;
+    }
+  }
+
+  void CheckOutputTimestamps() {
+    int num_full_packets = output().packets.size();
+    if (options_.pad_final_packet()) {
+      num_full_packets -= 1;
+    }
+
+    int64 num_samples = 0;
+    for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
+      const Packet& packet = output().packets[packet_num];
+      num_samples += FrameDurationSamples();
+      double expected_timestamp =
+          options_.use_local_timestamp()
+              ? GetExpectedLocalTimestampForSample(num_samples - 1)
+              : GetExpectedCumulativeTimestamp(num_samples - 1);
+      ASSERT_NEAR(packet.Timestamp().Seconds(), expected_timestamp, 1e-10);
+    }
+  }
+
+  ::mediapipe::Status RunTimestampTest() {
+    InitializeGraph();
+    InitializeInputForTimeStampingTest();
+    FillInputHeader();
+    return RunGraph();
+  }
+
+ private:
+  // Returns the timestamp in seconds based on local timestamping.
+  double GetExpectedLocalTimestampForSample(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / input_sample_rate_ +
+           (sample_index / kUniversalInputPacketSize) *
+               kGapBetweenPacketsInSeconds;
+  }
+
+  // Returns the timestamp inseconds based on cumulative timestamping.
+  double GetExpectedCumulativeTimestamp(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / FrameDurationSamples() * FrameDurationSamples() /
+               input_sample_rate_;
+  }
+};
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseLocalTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(true);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseCumulativeTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(false);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+}  // namespace
 }  // namespace mediapipe

mediapipe/calculators/core/BUILD

@@ -26,6 +26,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "dequantize_byte_array_calculator_proto",
+    srcs = ["dequantize_byte_array_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "packet_cloner_calculator_proto",
     srcs = ["packet_cloner_calculator.proto"],
@@ -76,7 +83,7 @@ mediapipe_cc_proto_library(
     name = "packet_cloner_calculator_cc_proto",
     srcs = ["packet_cloner_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":packet_cloner_calculator_proto"],
 )
 
@@ -84,7 +91,7 @@ mediapipe_cc_proto_library(
     name = "packet_resampler_calculator_cc_proto",
     srcs = ["packet_resampler_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":packet_resampler_calculator_proto"],
 )
 
@@ -92,7 +99,7 @@ mediapipe_cc_proto_library(
     name = "split_vector_calculator_cc_proto",
     srcs = ["split_vector_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":split_vector_calculator_proto"],
 )
 
@@ -104,11 +111,19 @@ mediapipe_cc_proto_library(
     deps = [":concatenate_vector_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "dequantize_byte_array_calculator_cc_proto",
+    srcs = ["dequantize_byte_array_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":dequantize_byte_array_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "quantize_float_vector_calculator_cc_proto",
     srcs = ["quantize_float_vector_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":quantize_float_vector_calculator_proto"],
 )
 
@@ -116,7 +131,7 @@ mediapipe_cc_proto_library(
     name = "sequence_shift_calculator_cc_proto",
     srcs = ["sequence_shift_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":sequence_shift_calculator_proto"],
 )
 
@@ -124,7 +139,7 @@ mediapipe_cc_proto_library(
     name = "gate_calculator_cc_proto",
     srcs = ["gate_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":gate_calculator_proto"],
 )
 
@@ -162,10 +177,17 @@ cc_library(
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@org_tensorflow//tensorflow/lite:framework",
-    ],
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+        ],
+    }),
     alwayslink = 1,
 )
 
@@ -380,6 +402,32 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "string_to_int_calculator",
+    srcs = ["string_to_int_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "side_packet_to_stream_calculator",
+    srcs = ["side_packet_to_stream_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "immediate_mux_calculator_test",
     srcs = ["immediate_mux_calculator_test.cc"],
@@ -523,6 +571,7 @@ cc_library(
     deps = [
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",
@@ -550,6 +599,32 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "dequantize_byte_array_calculator",
+    srcs = ["dequantize_byte_array_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":dequantize_byte_array_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "dequantize_byte_array_calculator_test",
+    srcs = ["dequantize_byte_array_calculator_test.cc"],
+    deps = [
+        ":dequantize_byte_array_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
 cc_library(
     name = "quantize_float_vector_calculator",
     srcs = ["quantize_float_vector_calculator.cc"],
@@ -627,6 +702,41 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "matrix_to_vector_calculator",
+    srcs = ["matrix_to_vector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:status_util",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "matrix_to_vector_calculator_test",
+    srcs = ["matrix_to_vector_calculator_test.cc"],
+    deps = [
+        ":matrix_to_vector_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+        "//mediapipe/util:time_series_test_util",
+        "//mediapipe/util:time_series_util",
+    ],
+)
+
 cc_library(
     name = "merge_calculator",
     srcs = ["merge_calculator.cc"],

mediapipe/calculators/core/add_header_calculator_test.cc

@@ -42,7 +42,7 @@ TEST_F(AddHeaderCalculatorTest, Works) {
   }
 
   // Run calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   ASSERT_EQ(1, runner.Outputs().NumEntries());
 
@@ -69,7 +69,7 @@ TEST_F(AddHeaderCalculatorTest, HandlesEmptyHeaderStream) {
 
   // No header and no packets.
   // Run calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   EXPECT_TRUE(runner.Outputs().Index(0).header.IsEmpty());
 }
 

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -16,8 +16,13 @@
 
 #include <vector>
 
+#include "mediapipe/framework/formats/landmark.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
 namespace mediapipe {
 
 // Example config:
@@ -41,4 +46,14 @@ typedef ConcatenateVectorCalculator<TfLiteTensor>
     ConcatenateTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
 
+typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
+    ConcatenateLandmarkVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
+    ConcatenateGlBufferVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
+#endif
+
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.h

@@ -15,6 +15,7 @@
 #ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_
 
+#include <type_traits>
 #include <vector>
 
 #include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
@@ -59,16 +60,58 @@ class ConcatenateVectorCalculator : public CalculatorBase {
         if (cc->Inputs().Index(i).IsEmpty()) return ::mediapipe::OkStatus();
       }
     }
-    auto output = absl::make_unique<std::vector<T>>();
+
+    return ConcatenateVectors<T>(std::is_copy_constructible<T>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConcatenateVectors(std::true_type,
+                                         CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
     for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
       if (cc->Inputs().Index(i).IsEmpty()) continue;
-      const std::vector<T>& input = cc->Inputs().Index(i).Get<std::vector<T>>();
+      const std::vector<U>& input = cc->Inputs().Index(i).Get<std::vector<U>>();
       output->insert(output->end(), input.begin(), input.end());
     }
     cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
     return ::mediapipe::OkStatus();
   }
 
+  template <typename U>
+  ::mediapipe::Status ConcatenateVectors(std::false_type,
+                                         CalculatorContext* cc) {
+    return ConsumeAndConcatenateVectors<T>(std::is_move_constructible<U>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndConcatenateVectors(std::true_type,
+                                                   CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      if (cc->Inputs().Index(i).IsEmpty()) continue;
+      ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+          cc->Inputs().Index(i).Value().Consume<std::vector<U>>();
+      if (input_status.ok()) {
+        std::unique_ptr<std::vector<U>> input_vector =
+            std::move(input_status).ValueOrDie();
+        output->insert(output->end(),
+                       std::make_move_iterator(input_vector->begin()),
+                       std::make_move_iterator(input_vector->end()));
+      } else {
+        return input_status.status();
+      }
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndConcatenateVectors(std::false_type,
+                                                   CalculatorContext* cc) {
+    return ::mediapipe::InternalError(
+        "Cannot copy or move input vectors to concatenate them");
+  }
+
  private:
   bool only_emit_if_all_present_;
 };

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -45,7 +45,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, EmptyVectorInputs) {
 
   std::vector<std::vector<int>> inputs = {{}, {}, {}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -60,7 +60,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, OneTimestamp) {
 
   std::vector<std::vector<int>> inputs = {{1, 2, 3}, {4}, {5, 6}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -81,7 +81,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
     std::vector<std::vector<int>> inputs = {{0, 2}, {1}, {3, 5}};
     AddInputVectors(inputs, /*timestamp=*/2, &runner);
   }
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(2, outputs.size());
@@ -106,7 +106,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
 
   std::vector<std::vector<int>> inputs = {{1, 2, 3}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -125,7 +125,7 @@ TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
 
   std::vector<std::vector<int>> inputs = {{1, 2, 3}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(0, outputs.size());
@@ -146,7 +146,7 @@ TEST(ConcatenateFloatVectorCalculatorTest, EmptyVectorInputs) {
 
   std::vector<std::vector<float>> inputs = {{}, {}, {}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -162,7 +162,7 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneTimestamp) {
   std::vector<std::vector<float>> inputs = {
       {1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -185,7 +185,7 @@ TEST(ConcatenateFloatVectorCalculatorTest, TwoInputsAtTwoTimestamps) {
         {0.0f, 2.0f}, {1.0f}, {3.0f, 5.0f}};
     AddInputVectors(inputs, /*timestamp=*/2, &runner);
   }
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(2, outputs.size());
@@ -210,7 +210,7 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamStillOutput) {
 
   std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(1, outputs.size());
@@ -229,10 +229,173 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
 
   std::vector<std::vector<float>> inputs = {{1.0f, 2.0f, 3.0f}};
   AddInputVectors(inputs, /*timestamp=*/1, &runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
   EXPECT_EQ(0, outputs.size());
 }
 
+typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
+    TestConcatenateUniqueIntPtrCalculator;
+REGISTER_CALCULATOR(TestConcatenateUniqueIntPtrCalculator);
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        input_stream: "in_3"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          input_stream: "in_3"
+          output_stream: "out"
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+  // input2: {3}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_2 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(1);
+  input_2->at(0) = absl::make_unique<int>(3);
+  // input3: {4, 5}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_3 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(2);
+  input_3->at(0) = absl::make_unique<int>(4);
+  input_3->at(1) = absl::make_unique<int>(5);
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_2", Adopt(input_2.release()).At(Timestamp(1))));
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_3", Adopt(input_3.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(6, result.size());
+  for (int i = 0; i < 6; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          output_stream: "out"
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(3, result.size());
+  for (int i = 0; i < 3; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "in_1"
+        input_stream: "in_2"
+        node {
+          calculator: "TestConcatenateUniqueIntPtrCalculator"
+          input_stream: "in_1"
+          input_stream: "in_2"
+          output_stream: "out"
+          options {
+            [mediapipe.ConcatenateVectorCalculatorOptions.ext] {
+              only_emit_if_all_present: true
+            }
+          }
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("out", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2}
+  std::unique_ptr<std::vector<std::unique_ptr<int>>> input_1 =
+      absl::make_unique<std::vector<std::unique_ptr<int>>>(3);
+  for (int i = 0; i < 3; ++i) {
+    input_1->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "in_1", Adopt(input_1.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(0, outputs.size());
+}
+
 }  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator.cc

@@ -0,0 +1,90 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cfloat>
+
+#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/status.h"
+
+// Dequantizes a byte array to a vector of floats.
+//
+// Example config:
+//   node {
+//     calculator: "DequantizeByteArrayCalculator"
+//     input_stream: "ENCODED:encoded"
+//     output_stream: "FLOAT_VECTOR:float_vector"
+//     options {
+//       [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+//         max_quantized_value: 2
+//         min_quantized_value: -2
+//       }
+//     }
+//   }
+namespace mediapipe {
+
+class DequantizeByteArrayCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag("ENCODED").Set<std::string>();
+    cc->Outputs().Tag("FLOAT_VECTOR").Set<std::vector<float>>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    const auto options =
+        cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>();
+    if (!options.has_max_quantized_value() ||
+        !options.has_min_quantized_value()) {
+      return ::mediapipe::InvalidArgumentError(
+          "Both max_quantized_value and min_quantized_value must be provided "
+          "in DequantizeByteArrayCalculatorOptions.");
+    }
+    float max_quantized_value = options.max_quantized_value();
+    float min_quantized_value = options.min_quantized_value();
+    if (max_quantized_value < min_quantized_value + FLT_EPSILON) {
+      return ::mediapipe::InvalidArgumentError(
+          "max_quantized_value must be greater than min_quantized_value.");
+    }
+    float range = max_quantized_value - min_quantized_value;
+    scalar_ = range / 255.0;
+    bias_ = (range / 512.0) + min_quantized_value;
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    const std::string& encoded =
+        cc->Inputs().Tag("ENCODED").Value().Get<std::string>();
+    std::vector<float> float_vector;
+    float_vector.reserve(encoded.length());
+    for (int i = 0; i < encoded.length(); ++i) {
+      float_vector.push_back(
+          static_cast<unsigned char>(encoded.at(i)) * scalar_ + bias_);
+    }
+    cc->Outputs()
+        .Tag("FLOAT_VECTOR")
+        .AddPacket(MakePacket<std::vector<float>>(float_vector)
+                       .At(cc->InputTimestamp()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float scalar_;
+  float bias_;
+};
+
+REGISTER_CALCULATOR(DequantizeByteArrayCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/dequantize_byte_array_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message DequantizeByteArrayCalculatorOptions {
+  extend CalculatorOptions {
+    optional DequantizeByteArrayCalculatorOptions ext = 272316343;
+  }
+
+  optional float max_quantized_value = 1;
+  optional float min_quantized_value = 2;
+}

mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc

@@ -0,0 +1,137 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "Both max_quantized_value and min_quantized_value must be provided"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: -2
+            min_quantized_value: 2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 1
+            min_quantized_value: 1
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  std::string empty_string;
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(empty_string).At(Timestamp(0)));
+  auto status = runner.Run();
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr(
+          "max_quantized_value must be greater than min_quantized_value"));
+}
+
+TEST(DequantizeByteArrayCalculatorTest, TestDequantization) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "DequantizeByteArrayCalculator"
+        input_stream: "ENCODED:encoded"
+        output_stream: "FLOAT_VECTOR:float_vector"
+        options {
+          [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: {
+            max_quantized_value: 2
+            min_quantized_value: -2
+          }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+  unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01};
+  runner.MutableInputs()->Tag("ENCODED").packets.push_back(
+      MakePacket<std::string>(
+          std::string(reinterpret_cast<char const*>(input), 4))
+          .At(Timestamp(0)));
+  auto status = runner.Run();
+  MP_ASSERT_OK(runner.Run());
+  const std::vector<Packet>& outputs =
+      runner.Outputs().Tag("FLOAT_VECTOR").packets;
+  EXPECT_EQ(1, outputs.size());
+  const std::vector<float>& result = outputs[0].Get<std::vector<float>>();
+  ASSERT_FALSE(result.empty());
+  EXPECT_EQ(4, result.size());
+  EXPECT_NEAR(0, result[0], 0.01);
+  EXPECT_NEAR(2, result[1], 0.01);
+  EXPECT_NEAR(-2, result[2], 0.01);
+  EXPECT_NEAR(-1.976, result[3], 0.01);
+
+  EXPECT_EQ(Timestamp(0), outputs[0].Timestamp());
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/flow_limiter_calculator_test.cc

@@ -91,7 +91,7 @@ TEST(FlowLimiterCalculator, OneOutputTest) {
   }
 
   // Run the calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& frame_output_packets =
       runner.Outputs().Index(0).packets;
 
@@ -117,7 +117,7 @@ TEST(FlowLimiterCalculator, BasicTest) {
   }
 
   // Run the calculator.
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const std::vector<Packet>& frame_output_packets =
       runner.Outputs().Index(0).packets;
 
@@ -198,7 +198,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
       close_count_++;
       return ::mediapipe::OkStatus();
     };
-    MEDIAPIPE_ASSERT_OK(graph_.Initialize(
+    MP_ASSERT_OK(graph_.Initialize(
         graph_config_, {
                            {"max_in_flight", MakePacket<int>(max_in_flight)},
                            {"callback_0", Adopt(new auto(semaphore_0_func))},
@@ -209,7 +209,7 @@ class FlowLimiterCalculatorTest : public testing::Test {
 
   // Adds a packet to a graph input stream.
   void AddPacket(const std::string& input_name, int value) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(value).At(Timestamp(value))));
   }
 
@@ -277,10 +277,10 @@ class FlowLimiterCalculatorTest : public testing::Test {
 //
 TEST_F(FlowLimiterCalculatorTest, BackEdgeCloses) {
   InitializeGraph(1);
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   auto send_packet = [this](const std::string& input_name, int64 n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int64>(n).At(Timestamp(n))));
   };
 
@@ -288,14 +288,14 @@ TEST_F(FlowLimiterCalculatorTest, BackEdgeCloses) {
     send_packet("in_1", i * 10);
     // This next input should be dropped.
     send_packet("in_1", i * 10 + 5);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
     send_packet("in_2", i * 10);
     exit_semaphore_.Release(1);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
   }
-  MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_1"));
+  MP_EXPECT_OK(graph_.CloseInputStream("in_1"));
-  MEDIAPIPE_EXPECT_OK(graph_.CloseInputStream("in_2"));
+  MP_EXPECT_OK(graph_.CloseInputStream("in_2"));
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
 
   // All output streams are closed and all output packets are delivered,
   // with stream "in_1" and stream "in_2" closed.
@@ -321,17 +321,17 @@ TEST_F(FlowLimiterCalculatorTest, BackEdgeCloses) {
 // input streams are closed after the last input packet has been processed.
 TEST_F(FlowLimiterCalculatorTest, AllStreamsClose) {
   InitializeGraph(1);
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   exit_semaphore_.Release(10);
   for (int i = 0; i < 10; i++) {
     AddPacket("in_1", i);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
     AddPacket("in_2", i);
-    MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+    MP_EXPECT_OK(graph_.WaitUntilIdle());
   }
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
 
   EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_));
   EXPECT_EQ(TimestampValues(out_1_packets_),
@@ -371,7 +371,7 @@ TEST(FlowLimiterCalculator, TwoStreams) {
   };
 
   CalculatorGraph graph_;
-  MEDIAPIPE_EXPECT_OK(graph_.Initialize(
+  MP_EXPECT_OK(graph_.Initialize(
       graph_config_,
       {
           {"max_in_flight", MakePacket<int>(1)},
@@ -379,63 +379,63 @@ TEST(FlowLimiterCalculator, TwoStreams) {
            MakePacket<std::function<void(const Packet&)>>(allow_cb)},
       }));
 
-  MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
+  MP_EXPECT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   send_packet("in_a", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(allow, false);
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{}));
 
   send_packet("in_a", 2);
   send_packet("in_b", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, false);
 
   send_packet("finished", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, true);
 
   send_packet("in_b", 2);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1}));
   EXPECT_EQ(allow, true);
 
   send_packet("in_b", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("in_b", 4);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("in_a", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, false);
 
   send_packet("finished", 3);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(a_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(TimestampValues(b_passed), (std::vector<int64>{1, 3}));
   EXPECT_EQ(allow, true);
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
 TEST(FlowLimiterCalculator, CanConsume) {
@@ -465,7 +465,7 @@ TEST(FlowLimiterCalculator, CanConsume) {
   };
 
   CalculatorGraph graph_;
-  MEDIAPIPE_EXPECT_OK(graph_.Initialize(
+  MP_EXPECT_OK(graph_.Initialize(
       graph_config_,
       {
           {"max_in_flight", MakePacket<int>(1)},
@@ -473,21 +473,21 @@ TEST(FlowLimiterCalculator, CanConsume) {
            MakePacket<std::function<void(const Packet&)>>(allow_cb)},
       }));
 
-  MEDIAPIPE_EXPECT_OK(graph_.StartRun({}));
+  MP_EXPECT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   send_packet("in", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(allow, false);
   EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector<int64>{1}));
 
-  MEDIAPIPE_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
+  MP_EXPECT_OK(in_sampled_packets_[0].Consume<int>());
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
 }  // anonymous namespace

mediapipe/calculators/core/gate_calculator_test.cc

@@ -32,7 +32,7 @@ class GateCalculatorTest : public ::testing::Test {
         ->Tag(control_tag)
         .packets.push_back(MakePacket<bool>(control).At(Timestamp(timestamp)));
 
-    MEDIAPIPE_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
+    MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed.";
   }
 
   void SetRunner(const std::string& proto) {

mediapipe/calculators/core/immediate_mux_calculator_test.cc

@@ -217,23 +217,23 @@ class ImmediateMuxCalculatorTest : public ::testing::Test {
 
     // Start running the graph.
     CalculatorGraph graph;
-    MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config_));
+    MP_ASSERT_OK(graph.Initialize(graph_config_));
-    MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+    MP_ASSERT_OK(graph.StartRun({}));
 
     // Send each packet to the graph in the specified order.
     for (int t = 0; t < input_sets.size(); t++) {
       const std::vector<Packet>& input_set = input_sets[t];
-      MEDIAPIPE_EXPECT_OK(graph.WaitUntilIdle());
+      MP_EXPECT_OK(graph.WaitUntilIdle());
       for (int i = 0; i < input_set.size(); i++) {
         const Packet& packet = input_set[i];
         if (!IsNone(packet)) {
-          MEDIAPIPE_EXPECT_OK(graph.AddPacketToInputStream(
+          MP_EXPECT_OK(graph.AddPacketToInputStream(
               absl::StrCat("input_packets_", i), packet));
         }
       }
     }
-    MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
+    MP_ASSERT_OK(graph.CloseAllInputStreams());
-    MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+    MP_ASSERT_OK(graph.WaitUntilDone());
   }
 
   CalculatorGraphConfig graph_config_;
@@ -335,22 +335,22 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
 
   // Start the graph and add five input packets.
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(
+  MP_ASSERT_OK(graph.Initialize(graph_config_,
-      graph_config_, {
+                                {
-                         {"callback_0", Adopt(new auto(wait_0))},
+                                    {"callback_0", Adopt(new auto(wait_0))},
-                         {"callback_1", Adopt(new auto(wait_1))},
+                                    {"callback_1", Adopt(new auto(wait_1))},
-                     }));
+                                }));
-  MEDIAPIPE_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
+  MP_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb));
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(10000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(20000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(30000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(40000)));
-  MEDIAPIPE_EXPECT_OK(
+  MP_EXPECT_OK(
       graph.AddPacketToInputStream("input_packets_0", PacketAt(50000)));
 
   // Release the outputs in order 20000, 10000, 30000, 50000, 40000.
@@ -362,8 +362,8 @@ TEST_F(ImmediateMuxCalculatorTest, Demux) {
   semaphore_0.Release(1);  // 50000
   wait_for([&] { return out_packets.size() >= 3; });
   semaphore_1.Release(1);  // 40000
-  MEDIAPIPE_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 
   // Output packets 10000 and 40000 are superseded and dropped.
   EXPECT_THAT(TimestampValues(out_packets), ElementsAre(20000, 30000, 50000));

mediapipe/calculators/core/matrix_multiply_calculator_test.cc

@@ -219,7 +219,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
         Adopt(sample).At(Timestamp(i)));
   }
 
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   EXPECT_EQ(runner.MutableInputs()->Index(0).packets.size(),
             runner.Outputs().Index(0).packets.size());
 

mediapipe/calculators/core/matrix_subtract_calculator_test.cc

@@ -112,7 +112,7 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromInput) {
   runner.MutableInputs()->Tag("MINUEND").packets.push_back(
       Adopt(input_matrix).At(Timestamp(0)));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
 
   EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());
@@ -142,7 +142,7 @@ TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) {
       ->Tag("SUBTRAHEND")
       .packets.push_back(Adopt(input_matrix).At(Timestamp(0)));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   EXPECT_EQ(1, runner.Outputs().Index(0).packets.size());
 
   EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp());

mediapipe/calculators/core/matrix_to_vector_calculator.cc

@@ -0,0 +1,83 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines MatrixToVectorCalculator.
+#include <math.h>
+
+#include <deque>
+#include <memory>
+#include <string>
+
+#include "Eigen/Core"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// A calculator that converts a Matrix M to a vector containing all the
+// entries of M in column-major order.
+//
+// Example config:
+// node {
+//   calculator: "MatrixToVectorCalculator"
+//   input_stream: "input_matrix"
+//   output_stream: "column_major_vector"
+// }
+class MatrixToVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input Packet containing a Matrix.
+    );
+    cc->Outputs().Index(0).Set<std::vector<float>>(
+        // Output Packet containing a vector, one for each input Packet.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Outputs a packet containing a vector for each input packet.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(MatrixToVectorCalculator);
+
+::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
+  // Inform the framework that we don't alter timestamps.
+  cc->SetOffset(mediapipe::TimestampDiff(0));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
+  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+  auto output = absl::make_unique<std::vector<float>>();
+
+  // The following lines work to convert the Matrix to a vector because Matrix
+  // is an Eigen::MatrixXf and Eigen uses column-major layout by default.
+  output->resize(input.rows() * input.cols());
+  auto output_as_matrix =
+      Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
+  output_as_matrix = input;
+
+  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_to_vector_calculator_test.cc

@@ -0,0 +1,88 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "mediapipe/util/time_series_test_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+namespace {
+
+class MatrixToVectorCalculatorTest
+    : public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
+ protected:
+  void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
+
+  void AppendInput(const std::vector<float>& column_major_data,
+                   int64 timestamp) {
+    ASSERT_EQ(num_input_samples_ * num_input_channels_,
+              column_major_data.size());
+    Eigen::Map<const Matrix> data_map(&column_major_data[0],
+                                      num_input_channels_, num_input_samples_);
+    AppendInputPacket(new Matrix(data_map), timestamp);
+  }
+
+  void SetInputStreamParameters(int num_channels, int num_samples) {
+    num_input_channels_ = num_channels;
+    num_input_samples_ = num_samples;
+    input_sample_rate_ = 100;
+    input_packet_rate_ = 20.0;
+  }
+
+  void SetInputHeader(int num_channels, int num_samples) {
+    SetInputStreamParameters(num_channels, num_samples);
+    FillInputHeader();
+  }
+
+  void CheckOutputPacket(int packet, std::vector<float> expected_vector) {
+    const auto& actual_vector =
+        runner_->Outputs().Index(0).packets[packet].Get<std::vector<float>>();
+    EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector));
+  }
+};
+
+TEST_F(MatrixToVectorCalculatorTest, SingleRow) {
+  InitializeGraph();
+  SetInputHeader(1, 4);  // 1 channel x 4 samples
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0};
+  AppendInput(data_vector, 0);
+  MP_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) {
+  InitializeGraph();
+  SetInputHeader(4, 2);  // 4 channels x 2 samples
+  // Actual data matrix is the transpose of the appearance below.
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0,
+                                           5.0, 6.0, 7.0, 8.0};
+  AppendInput(data_vector, 0);
+
+  MP_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/core/merge_calculator_test.cc

@@ -78,7 +78,7 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
   runner.MutableInputs()->Index(1).packets.push_back(
       Adopt(new float(35.5)).At(Timestamp(35)));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   // Expected combined_output: 5.5, 10, 20, 30, 35.5 at times 5, 10, 20, 30, 35.
   const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;
@@ -120,7 +120,7 @@ TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest,
   runner.MutableInputs()->Index(2).packets.push_back(
       Adopt(new char('c')).At(Timestamp(10)));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   // Expected combined_output: 'c', 20.5, 30 at times 10, 20, 30.
   const std::vector<Packet>& actual_output = runner.Outputs().Index(0).packets;

mediapipe/calculators/core/packet_inner_join_calculator_test.cc

@@ -37,7 +37,7 @@ TEST(PacketInnerJoinCalculatorTest, AllMatching) {
   for (int packet_load : packets_on_stream2) {
     runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
   }
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // Check.
   const std::vector<int> expected = {0, 1, 2, 3};
   ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());
@@ -64,7 +64,7 @@ TEST(PacketInnerJoinCalculatorTest, NoneMatching) {
   for (int packet_load : packets_on_stream2) {
     runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
   }
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // Check.
   EXPECT_TRUE(runner.Outputs().Index(0).packets.empty());
   EXPECT_TRUE(runner.Outputs().Index(1).packets.empty());
@@ -82,7 +82,7 @@ TEST(PacketInnerJoinCalculatorTest, SomeMatching) {
   for (int packet_load : packets_on_stream2) {
     runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load));
   }
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // Check.
   const std::vector<int> expected = {0, 2, 4, 6};
   ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size());

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -287,9 +287,9 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
     }
   }
   if (jitter_ != 0.0 && random_ != nullptr) {
-    RETURN_IF_ERROR(ProcessWithJitter(cc));
+    MP_RETURN_IF_ERROR(ProcessWithJitter(cc));
   } else {
-    RETURN_IF_ERROR(ProcessWithoutJitter(cc));
+    MP_RETURN_IF_ERROR(ProcessWithoutJitter(cc));
   }
   last_packet_ = cc->Inputs().Get(input_data_id_).Value();
   return ::mediapipe::OkStatus();

mediapipe/calculators/core/packet_resampler_calculator_test.cc

@@ -103,7 +103,7 @@ TEST(PacketResamplerCalculatorTest, NoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
   }
 }
 
@@ -114,7 +114,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0}, {0});
   }
 
@@ -124,7 +124,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({1000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({1000}, {1000});
   }
 
@@ -134,7 +134,7 @@ TEST(PacketResamplerCalculatorTest, SinglePacketInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({16668});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({16668}, {16668});
   }
 }
@@ -146,7 +146,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16666});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0}, {0});
   }
 
@@ -156,7 +156,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 16667}, {0, 33333});
   }
 
@@ -166,7 +166,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 49999});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 49999}, {0, 33333});
   }
 
@@ -176,7 +176,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 50000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 0, 50000}, {0, 33333, 66667});
   }
 
@@ -186,7 +186,7 @@ TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({2000, 118666});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({2000, 2000, 2000, 118666},
                                  {2000, 35333, 68667, 102000});
   }
@@ -197,7 +197,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepoints) {
       "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
       "{frame_rate:30}");
   runner.SetInput({0, 33333, 66667, 100000, 133333, 166667, 200000});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   runner.CheckOutputTimestamps(
       {0, 33333, 66667, 100000, 133333, 166667, 200000},
       {0, 33333, 66667, 100000, 133333, 166667, 200000});
@@ -210,7 +210,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriods) {
       "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
       "{frame_rate:30}");
   runner.SetInput({0, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   runner.CheckOutputTimestamps({0, 33300, 66600}, {0, 33333, 66667});
 }
 
@@ -222,7 +222,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 5000, 16666, 83334});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 16666, 16666, 83334},
                                  {0, 33333, 66667, 100000});
   }
@@ -232,7 +232,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 16666, 16667, 25000, 33000, 35000, 135000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 33000, 35000, 35000, 135000},
                                  {0, 33333, 66667, 100000, 133333});
   }
@@ -242,7 +242,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({0, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -255,7 +255,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:500000}");
     runner.SetInput({0, 10, 13});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 0, 0, 0, 0, 10, 10, 13},
                                  {0, 2, 4, 6, 8, 10, 12, 14});
   }
@@ -266,7 +266,7 @@ TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:1000000}");
     runner.SetInput({0, 10, 13});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps(
         {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 13},
         {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13});
@@ -280,7 +280,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-200, -20, 16466});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-200}, {-200});
   }
 
@@ -290,7 +290,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-200, -20, 16467});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-200, 16467}, {-200, 33133});
   }
 
@@ -300,7 +300,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-500, 66667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-500, -500, 66667}, {-500, 32833, 66167});
   }
 
@@ -310,7 +310,7 @@ TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-50000, -33334, 33334});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-50000, -33334, -33334, 33334},
                                  {-50000, -16667, 16667, 50000});
   }
@@ -323,7 +323,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:50}");
     runner.SetInput({0, 9999, 29999});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 29999}, {0, 20000});
   }
 
@@ -333,7 +333,7 @@ TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:50}");
     runner.SetInput({0, 10000, 50000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 10000, 10000, 50000},
                                  {0, 20000, 40000, 60000});
   }
@@ -347,7 +347,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
     runner.SetInput({0, 10000, 30000, 50000, 60000});
     runner.SetVideoHeader(50.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 10000, 30000, 60000},
                                  {0, 20000, 40000, 60000});
     runner.CheckVideoHeader(50.0);
@@ -360,7 +360,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}");
     runner.SetInput({0, 5000, 10010, 15001, 19990});
     runner.SetVideoHeader(200.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
     runner.CheckVideoHeader(50.0);
   }
@@ -372,7 +372,7 @@ TEST(PacketResamplerCalculatorTest, FrameRateTest) {
         "{frame_rate:50, output_header:PASS_HEADER}");
     runner.SetInput({0, 5000, 10010, 15001, 19990});
     runner.SetVideoHeader(200.0);
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({0, 19990}, {0, 20000});
     runner.CheckVideoHeader(200.0);
   }
@@ -404,7 +404,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
       ->Tag("VIDEO_HEADER")
       .packets.push_back(
           Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream()));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
 
   ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size());
   EXPECT_EQ(Timestamp::PreStream(),
@@ -424,7 +424,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithoutRound) {
         frame_rate: 1
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1333333 is not emitted as 2000000, because it does not round to 2000000.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -435,7 +435,7 @@ TEST(PacketResamplerCalculatorTest, FlushLastPacketWithRound) {
         frame_rate: 1
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1666667 is emitted as 2000000, because it rounds to 2000000.
   runner.CheckOutputTimestamps({0, 1000000, 1666667}, {0, 1000000, 2000000});
 }
@@ -447,7 +447,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithoutRound) {
         flush_last_packet: false
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1333333 is not emitted no matter what; see FlushLastPacketWithoutRound.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -459,7 +459,7 @@ TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithRound) {
         flush_last_packet: false
       })");
   runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667});
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   // 1666667 is not emitted due to flush_last_packet: false.
   runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000});
 }
@@ -473,7 +473,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({33111, 66667, 100000, 133333, 166667, 200000},
                                  {33111, 66444, 99778, 133111, 166444, 199778});
   }
@@ -484,7 +484,7 @@ TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps(
         {33111, 66667, 100000, 133333, 166667, 200000},
         {33333, 66666, 100000, 133333, 166666, 200000});
@@ -499,7 +499,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 33300, 66600}, {-222, 33111, 66445});
   }
   {
@@ -509,7 +509,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "{frame_rate:30 "
         "base_timestamp:900011}");
     runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 33300, 66600}, {11, 33344, 66678});
   }
   {
@@ -521,7 +521,7 @@ TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) {
         "base_timestamp:11}");
     runner.SetInput(
         {899888, 916666, 916667, 920000, 933300, 949999, 950000, 966600});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({899888, 933300, 966600},
                                  {900011, 933344, 966678});
   }
@@ -536,7 +536,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
         "[mediapipe.PacketResamplerCalculatorOptions.ext]: "
         "{frame_rate:30}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {-222, 33111, 66445, 99778, 133111, 166445});
   }
@@ -547,7 +547,7 @@ TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -565,7 +565,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({66667, 100020, 133333, 166667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({66667, 100020, 133333, 166667},
                                  {66667, 100000, 133334, 166667});
   }
@@ -582,7 +582,7 @@ TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({100020, 133333, 166667});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({100020, 133333, 166667},
                                  {100000, 133333, 166667});
   }
@@ -596,7 +596,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "{frame_rate:30 "
         "base_timestamp:0}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000},
                                  {0, 33333, 66667, 100000, 133333, 166667});
   }
@@ -609,7 +609,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "start_time:40000 "
         "end_time:160000}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({49999, 49999, 49999},
                                  {66667, 100000, 133333});
   }
@@ -624,7 +624,7 @@ TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) {
         "end_time:160000 "
         "round_limits:true}");
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     runner.CheckOutputTimestamps({32000, 49999, 49999, 49999, 150000},
                                  {33333, 66667, 100000, 133333, 166667});
   }
@@ -654,7 +654,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
               })"));
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
   }
   {
@@ -670,7 +670,7 @@ TEST(PacketResamplerCalculatorTest, OptionsSidePacket) {
     runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options);
 
     runner.SetInput({-222, 15000, 32000, 49999, 150000});
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     EXPECT_EQ(6, runner.Outputs().Index(0).packets.size());
   }
 }

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -102,6 +102,12 @@ class PreviousLoopbackCalculator : public CalculatorBase {
         cc->Outputs().Get(loop_out_id_).AddPacket(std::move(previous_loopback));
       }
     }
+    if (!main_ts_.empty()) {
+      cc->Outputs().Get(loop_out_id_).SetNextTimestampBound(main_ts_.front());
+    }
+    if (cc->Inputs().Get(main_id_).IsDone() && main_ts_.empty()) {
+      cc->Outputs().Get(loop_out_id_).Close();
+    }
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/core/previous_loopback_calculator_test.cc

@@ -74,11 +74,11 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   tool::AddVectorSink("pair", &graph_config_, &in_prev);
 
   CalculatorGraph graph_;
-  MEDIAPIPE_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
-  MEDIAPIPE_ASSERT_OK(graph_.StartRun({}));
+  MP_ASSERT_OK(graph_.StartRun({}));
 
   auto send_packet = [&graph_](const std::string& input_name, int n) {
-    MEDIAPIPE_EXPECT_OK(graph_.AddPacketToInputStream(
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
         input_name, MakePacket<int>(n).At(Timestamp(n))));
   };
   auto pair_values = [](const Packet& packet) {
@@ -89,22 +89,113 @@ TEST(PreviousLoopbackCalculator, CorrectTimestamps) {
   };
 
   send_packet("in", 1);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(1, -1));
 
   send_packet("in", 5);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(5, 1));
 
   send_packet("in", 15);
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilIdle());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
   EXPECT_EQ(TimestampValues(in_prev), (std::vector<int64>{1, 5, 15}));
   EXPECT_EQ(pair_values(in_prev.back()), std::make_pair(15, 5));
 
-  MEDIAPIPE_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
-  MEDIAPIPE_EXPECT_OK(graph_.WaitUntilDone());
+  MP_EXPECT_OK(graph_.WaitUntilDone());
+}
+
+// A Calculator that outputs a summary packet in CalculatorBase::Close().
+class PacketOnCloseCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    sum_ += cc->Inputs().Index(0).Value().Get<int>();
+    cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Close(CalculatorContext* cc) final {
+    cc->Outputs().Index(0).AddPacket(
+        MakePacket<int>(sum_).At(Timestamp::Max()));
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  int sum_ = 0;
+};
+REGISTER_CALCULATOR(PacketOnCloseCalculator);
+
+// Demonstrates that all ouput and input streams in PreviousLoopbackCalculator
+// will close as expected when all graph input streams are closed.
+TEST(PreviousLoopbackCalculator, ClosesCorrectly) {
+  std::vector<Packet> outputs;
+  CalculatorGraphConfig graph_config_ =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: 'in'
+        node {
+          calculator: 'PreviousLoopbackCalculator'
+          input_stream: 'MAIN:in'
+          input_stream: 'LOOP:out'
+          input_stream_info: { tag_index: 'LOOP' back_edge: true }
+          output_stream: 'PREV_LOOP:previous'
+        }
+        # This calculator synchronizes its inputs as normal, so it is used
+        # to check that both "in" and "previous" are ready.
+        node {
+          calculator: 'PassThroughCalculator'
+          input_stream: 'in'
+          input_stream: 'previous'
+          output_stream: 'out'
+          output_stream: 'previous2'
+        }
+        node {
+          calculator: 'PacketOnCloseCalculator'
+          input_stream: 'out'
+          output_stream: 'close_out'
+        }
+      )");
+  tool::AddVectorSink("close_out", &graph_config_, &outputs);
+
+  CalculatorGraph graph_;
+  MP_ASSERT_OK(graph_.Initialize(graph_config_, {}));
+  MP_ASSERT_OK(graph_.StartRun({}));
+
+  auto send_packet = [&graph_](const std::string& input_name, int n) {
+    MP_EXPECT_OK(graph_.AddPacketToInputStream(
+        input_name, MakePacket<int>(n).At(Timestamp(n))));
+  };
+
+  send_packet("in", 1);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1}));
+
+  send_packet("in", 5);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5}));
+
+  send_packet("in", 15);
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs), (std::vector<int64>{1, 5, 15}));
+
+  MP_EXPECT_OK(graph_.CloseAllInputStreams());
+  MP_EXPECT_OK(graph_.WaitUntilIdle());
+  EXPECT_EQ(TimestampValues(outputs),
+            (std::vector<int64>{1, 5, 15, Timestamp::Max().Value()}));
+
+  MP_EXPECT_OK(graph_.WaitUntilDone());
 }
 
 }  // anonymous namespace

mediapipe/calculators/core/quantize_float_vector_calculator_test.cc

@@ -124,7 +124,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) {
       ->Tag("FLOAT_VECTOR")
       .packets.push_back(
           MakePacket<std::vector<float>>(empty_vector).At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
   EXPECT_EQ(1, outputs.size());
   EXPECT_TRUE(outputs[0].Get<std::string>().empty());
@@ -150,7 +150,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) {
       ->Tag("FLOAT_VECTOR")
       .packets.push_back(
           MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
   EXPECT_EQ(1, outputs.size());
   const std::string& result = outputs[0].Get<std::string>();
@@ -188,7 +188,7 @@ TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) {
       ->Tag("FLOAT_VECTOR")
       .packets.push_back(
           MakePacket<std::vector<float>>(vector).At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& outputs = runner.Outputs().Tag("ENCODED").packets;
   EXPECT_EQ(1, outputs.size());
   const std::string& result = outputs[0].Get<std::string>();

mediapipe/calculators/core/sequence_shift_calculator_test.cc

@@ -38,7 +38,7 @@ TEST(SequenceShiftCalculatorTest, ZeroShift) {
       "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 0 }", 1,
       1, 0);
   AddPackets(&runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& input_packets =
       runner.MutableInputs()->Index(0).packets;
   const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
@@ -59,7 +59,7 @@ TEST(SequenceShiftCalculatorTest, PositiveShift) {
       "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 3 }", 1,
       1, 0);
   AddPackets(&runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& input_packets =
       runner.MutableInputs()->Index(0).packets;
   const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;
@@ -83,7 +83,7 @@ TEST(SequenceShiftCalculatorTest, NegativeShift) {
       "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: -2 }",
       1, 1, 0);
   AddPackets(&runner);
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const std::vector<Packet>& input_packets =
       runner.MutableInputs()->Index(0).packets;
   const std::vector<Packet>& output_packets = runner.Outputs().Index(0).packets;

mediapipe/calculators/core/side_packet_to_stream_calculator.cc

@@ -0,0 +1,83 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+using mediapipe::PacketTypeSet;
+using mediapipe::Timestamp;
+
+namespace {
+static std::map<std::string, Timestamp>* kTimestampMap = []() {
+  auto* res = new std::map<std::string, Timestamp>();
+  res->emplace("AT_PRESTREAM", Timestamp::PreStream());
+  res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
+  res->emplace("AT_ZERO", Timestamp(0));
+  return res;
+}();
+
+}  // namespace
+
+// Outputs the single input_side_packet at the timestamp specified in the
+// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
+class SidePacketToStreamCalculator : public CalculatorBase {
+ public:
+  SidePacketToStreamCalculator() = default;
+  ~SidePacketToStreamCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(SidePacketToStreamCalculator);
+
+::mediapipe::Status SidePacketToStreamCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->InputSidePackets().Index(0).SetAny();
+
+  std::set<std::string> tags = cc->Outputs().GetTags();
+  RET_CHECK_EQ(tags.size(), 1);
+
+  RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
+  cc->Outputs().Tag(*tags.begin()).SetAny();
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status SidePacketToStreamCalculator::Process(
+    CalculatorContext* cc) {
+  return mediapipe::tool::StatusStop();
+}
+
+::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
+  std::set<std::string> tags = cc->Outputs().GetTags();
+  RET_CHECK_EQ(tags.size(), 1);
+  const std::string& tag = *tags.begin();
+  RET_CHECK_EQ(kTimestampMap->count(tag), 1);
+  cc->Outputs().Tag(tag).AddPacket(
+      cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
+
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -16,6 +16,7 @@
 
 #include <vector>
 
+#include "mediapipe/framework/formats/landmark.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
@@ -37,4 +38,7 @@ namespace mediapipe {
 typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
 
+typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
+    SplitLandmarkVectorCalculator;
+REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.h

@@ -34,7 +34,9 @@ namespace mediapipe {
 // SplitVectorCalculatorOptions. If the option "element_only" is set to true,
 // all ranges should be of size 1 and all outputs will be elements of type T. If
 // "element_only" is false, ranges can be non-zero in size and all outputs will
-// be of type std::vector<T>.
+// be of type std::vector<T>. If the option "combine_outputs" is set to true,
+// only one output stream can be specified and all ranges of elements will be
+// combined into one vector.
 // To use this class for a particular type T, register a calculator using
 // SplitVectorCalculator<T>.
 template <typename T>
@@ -49,28 +51,47 @@ class SplitVectorCalculator : public CalculatorBase {
     const auto& options =
         cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
 
-    if (cc->Outputs().NumEntries() != options.ranges_size()) {
+    if (options.combine_outputs()) {
-      return ::mediapipe::InvalidArgumentError(
+      RET_CHECK_EQ(cc->Outputs().NumEntries(), 1);
-          "The number of output streams should match the number of ranges "
+      cc->Outputs().Index(0).Set<std::vector<T>>();
-          "specified in the CalculatorOptions.");
+      for (int i = 0; i < options.ranges_size() - 1; ++i) {
-    }
+        for (int j = i + 1; j < options.ranges_size(); ++j) {
-
+          const auto& range_0 = options.ranges(i);
-    // Set the output types for each output stream.
+          const auto& range_1 = options.ranges(j);
-    for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+          if ((range_0.begin() >= range_1.begin() &&
-      if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+               range_0.begin() < range_1.end()) ||
-          options.ranges(i).begin() >= options.ranges(i).end()) {
+              (range_1.begin() >= range_0.begin() &&
+               range_1.begin() < range_0.end())) {
+            return ::mediapipe::InvalidArgumentError(
+                "Ranges must be non-overlapping when using combine_outputs "
+                "option.");
+          }
+        }
+      }
+    } else {
+      if (cc->Outputs().NumEntries() != options.ranges_size()) {
         return ::mediapipe::InvalidArgumentError(
-            "Indices should be non-negative and begin index should be less "
+            "The number of output streams should match the number of ranges "
-            "than the end index.");
+            "specified in the CalculatorOptions.");
       }
-      if (options.element_only()) {
+
-        if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+      // Set the output types for each output stream.
+      for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+        if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 ||
+            options.ranges(i).begin() >= options.ranges(i).end()) {
           return ::mediapipe::InvalidArgumentError(
-              "Since element_only is true, all ranges should be of size 1.");
+              "Indices should be non-negative and begin index should be less "
+              "than the end index.");
+        }
+        if (options.element_only()) {
+          if (options.ranges(i).end() - options.ranges(i).begin() != 1) {
+            return ::mediapipe::InvalidArgumentError(
+                "Since element_only is true, all ranges should be of size 1.");
+          }
+          cc->Outputs().Index(i).Set<T>();
+        } else {
+          cc->Outputs().Index(i).Set<std::vector<T>>();
         }
-        cc->Outputs().Index(i).Set<T>();
-      } else {
-        cc->Outputs().Index(i).Set<std::vector<T>>();
       }
     }
 
@@ -83,13 +104,15 @@ class SplitVectorCalculator : public CalculatorBase {
     const auto& options =
         cc->Options<::mediapipe::SplitVectorCalculatorOptions>();
 
+    element_only_ = options.element_only();
+    combine_outputs_ = options.combine_outputs();
+
     for (const auto& range : options.ranges()) {
       ranges_.push_back({range.begin(), range.end()});
       max_range_end_ = std::max(max_range_end_, range.end());
+      total_elements_ += range.end() - range.begin();
     }
 
-    element_only_ = options.element_only();
-
     return ::mediapipe::OkStatus();
   }
 
@@ -97,17 +120,29 @@ class SplitVectorCalculator : public CalculatorBase {
     const auto& input = cc->Inputs().Index(0).Get<std::vector<T>>();
     RET_CHECK_GE(input.size(), max_range_end_);
 
-    if (element_only_) {
+    if (combine_outputs_) {
+      auto output = absl::make_unique<std::vector<T>>();
+      output->reserve(total_elements_);
       for (int i = 0; i < ranges_.size(); ++i) {
-        cc->Outputs().Index(i).AddPacket(
+        auto elements = absl::make_unique<std::vector<T>>(
-            MakePacket<T>(input[ranges_[i].first]).At(cc->InputTimestamp()));
-      }
-    } else {
-      for (int i = 0; i < ranges_.size(); ++i) {
-        auto output = absl::make_unique<std::vector<T>>(
             input.begin() + ranges_[i].first,
             input.begin() + ranges_[i].second);
-        cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
+        output->insert(output->end(), elements->begin(), elements->end());
+      }
+      cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    } else {
+      if (element_only_) {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          cc->Outputs().Index(i).AddPacket(
+              MakePacket<T>(input[ranges_[i].first]).At(cc->InputTimestamp()));
+        }
+      } else {
+        for (int i = 0; i < ranges_.size(); ++i) {
+          auto output = absl::make_unique<std::vector<T>>(
+              input.begin() + ranges_[i].first,
+              input.begin() + ranges_[i].second);
+          cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp());
+        }
       }
     }
 
@@ -117,7 +152,9 @@ class SplitVectorCalculator : public CalculatorBase {
  private:
   std::vector<std::pair<int32, int32>> ranges_;
   int32 max_range_end_ = -1;
+  int32 total_elements_ = 0;
   bool element_only_ = false;
+  bool combine_outputs_ = false;
 };
 
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.proto

@@ -37,4 +37,7 @@ message SplitVectorCalculatorOptions {
   // just element of type T. By default, if a range specifies only one element,
   // it is outputted as an std::vector<T>.
   optional bool element_only = 2 [default = false];
+
+  // Combines output elements to one vector.
+  optional bool combine_outputs = 3 [default = false];
 }

mediapipe/calculators/core/split_vector_calculator_test.cc

@@ -105,6 +105,34 @@ class SplitTfLiteTensorVectorCalculatorTest : public ::testing::Test {
     }
   }
 
+  void ValidateCombinedVectorOutput(std::vector<Packet>& output_packets,
+                                    int expected_elements,
+                                    std::vector<int>& input_begin_indices,
+                                    std::vector<int>& input_end_indices) {
+    ASSERT_EQ(1, output_packets.size());
+    ASSERT_EQ(input_begin_indices.size(), input_end_indices.size());
+    const std::vector<TfLiteTensor>& output_vec =
+        output_packets[0].Get<std::vector<TfLiteTensor>>();
+    ASSERT_EQ(expected_elements, output_vec.size());
+    const int num_ranges = input_begin_indices.size();
+
+    int element_id = 0;
+    for (int range_id = 0; range_id < num_ranges; ++range_id) {
+      for (int i = input_begin_indices[range_id];
+           i < input_end_indices[range_id]; ++i) {
+        const int expected_value = i;
+        const TfLiteTensor* result = &output_vec[element_id];
+        float* result_buffer = result->data.f;
+        ASSERT_NE(result_buffer, nullptr);
+        ASSERT_EQ(result_buffer, input_buffers_[i]);
+        for (int j = 0; j < width * height * channels; ++j) {
+          ASSERT_EQ(expected_value, result_buffer[j]);
+        }
+        element_id++;
+      }
+    }
+  }
+
   void ValidateElementOutput(std::vector<Packet>& output_packets,
                              int input_begin_index) {
     ASSERT_EQ(1, output_packets.size());
@@ -161,12 +189,12 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
 
   // Run the graph.
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.Initialize(graph_config));
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
   // Wait until the calculator finishes processing.
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
 
   ValidateVectorOutput(range_0_packets, /*expected_elements=*/1,
                        /*input_begin_index=*/0);
@@ -176,8 +204,8 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) {
                        /*input_begin_index=*/4);
 
   // Fully close the graph at the end.
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) {
@@ -234,6 +262,65 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) {
   ASSERT_FALSE(graph.Initialize(graph_config).ok());
 }
 
+TEST_F(SplitTfLiteTensorVectorCalculatorTest,
+       InvalidCombineOutputsMultipleOutputsTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              output_stream: "range_1"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because the number of output streams does not
+  // match the number of range elements in the options.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 3 }
+                  ranges: { begin: 1 end: 4 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+
+  // Run the graph.
+  CalculatorGraph graph;
+  // The graph should fail running because there are overlapping ranges.
+  ASSERT_FALSE(graph.Initialize(graph_config).ok());
+}
+
 TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
   ASSERT_NE(interpreter_, nullptr);
 
@@ -270,12 +357,12 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
 
   // Run the graph.
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.Initialize(graph_config));
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
   // Wait until the calculator finishes processing.
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
 
   ValidateElementOutput(range_0_packets,
                         /*input_begin_index=*/0);
@@ -285,8 +372,55 @@ TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) {
                         /*input_begin_index=*/4);
 
   // Fully close the graph at the end.
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) {
+  ASSERT_NE(interpreter_, nullptr);
+
+  PrepareTfLiteTensorVector(/*vector_size=*/5);
+  ASSERT_NE(input_vec_, nullptr);
+
+  // Prepare a graph to use the SplitTfLiteTensorVectorCalculator.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tensor_in"
+            node {
+              calculator: "SplitTfLiteTensorVectorCalculator"
+              input_stream: "tensor_in"
+              output_stream: "range_0"
+              options {
+                [mediapipe.SplitVectorCalculatorOptions.ext] {
+                  ranges: { begin: 0 end: 1 }
+                  ranges: { begin: 2 end: 3 }
+                  ranges: { begin: 4 end: 5 }
+                  combine_outputs: true
+                }
+              }
+            }
+          )");
+  std::vector<Packet> range_0_packets;
+  tool::AddVectorSink("range_0", &graph_config, &range_0_packets);
+
+  // Run the graph.
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0))));
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  std::vector<int> input_begin_indices = {0, 2, 4};
+  std::vector<int> input_end_indices = {1, 3, 5};
+  ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3,
+                               input_begin_indices, input_end_indices);
+
+  // Fully close the graph at the end.
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 TEST_F(SplitTfLiteTensorVectorCalculatorTest,

mediapipe/calculators/core/string_to_int_calculator.cc

@@ -0,0 +1,79 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/types.h>
+
+#include <memory>
+#include <string>
+
+#include "absl/strings/numbers.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator that converts a std::string into an integer type, or fails if the
+// conversion is not possible.
+//
+// Example config:
+// node {
+//   calculator: "StringToIntCalculator"
+//   input_side_packet: "string"
+//   output_side_packet: "index"
+// }
+template <typename IntType>
+class StringToIntCalculatorTemplate : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets().Index(0).Set<std::string>();
+    cc->OutputSidePackets().Index(0).Set<IntType>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    IntType number;
+    if (!absl::SimpleAtoi(cc->InputSidePackets().Index(0).Get<std::string>(),
+                          &number)) {
+      return ::mediapipe::InvalidArgumentError(
+          "The std::string could not be parsed as an integer.");
+    }
+    cc->OutputSidePackets().Index(0).Set(MakePacket<IntType>(number));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+};
+
+using StringToIntCalculator = StringToIntCalculatorTemplate<int>;
+REGISTER_CALCULATOR(StringToIntCalculator);
+
+using StringToUintCalculator = StringToIntCalculatorTemplate<uint>;
+REGISTER_CALCULATOR(StringToUintCalculator);
+
+using StringToInt32Calculator = StringToIntCalculatorTemplate<int32>;
+REGISTER_CALCULATOR(StringToInt32Calculator);
+
+using StringToUint32Calculator = StringToIntCalculatorTemplate<uint32>;
+REGISTER_CALCULATOR(StringToUint32Calculator);
+
+using StringToInt64Calculator = StringToIntCalculatorTemplate<int64>;
+REGISTER_CALCULATOR(StringToInt64Calculator);
+
+using StringToUint64Calculator = StringToIntCalculatorTemplate<uint64>;
+REGISTER_CALCULATOR(StringToUint64Calculator);
+
+}  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -19,7 +19,6 @@ package(default_visibility = ["//visibility:private"])
 exports_files(["LICENSE"])
 
 load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-load("@bazel_skylib//lib:selects.bzl", "selects")
 
 proto_library(
     name = "opencv_image_encoder_calculator_proto",
@@ -81,7 +80,7 @@ mediapipe_cc_proto_library(
     name = "opencv_image_encoder_calculator_cc_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
@@ -89,7 +88,7 @@ mediapipe_cc_proto_library(
     name = "mask_overlay_calculator_cc_proto",
     srcs = ["mask_overlay_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":mask_overlay_calculator_proto"],
 )
 
@@ -100,7 +99,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":scale_image_calculator_proto"],
 )
 
@@ -110,7 +109,7 @@ mediapipe_cc_proto_library(
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":set_alpha_calculator_proto"],
 )
 
@@ -120,17 +119,17 @@ mediapipe_cc_proto_library(
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":image_cropping_calculator_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "bilateral_filter_calculator_cc_proto",
     srcs = ["bilateral_filter_calculator.proto"],
-    cc_deps = [
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-        "//mediapipe/framework:calculator_cc_proto",
+    visibility = [
+        "//visibility:public",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
     deps = [":bilateral_filter_calculator_proto"],
 )
 
@@ -141,7 +140,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/util:color_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":recolor_calculator_proto"],
 )
 
@@ -227,19 +226,13 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//mediapipe:ios": [
-            "//mediapipe/gpu:gl_calculator_helper",
-            "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
-            "//mediapipe/gpu:shader_util",
-        ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -263,13 +256,13 @@ cc_library(
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -291,7 +284,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/gpu:scale_mode_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":image_transformation_calculator_proto"],
 )
 
@@ -322,14 +315,14 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-    ] + selects.with_or({
+    ] + select({
-        ("//mediapipe:android", "//mediapipe:ios"): [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -363,14 +356,15 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-    ] + selects.with_or({
+        "//mediapipe/gpu:gpu_buffer",
-        ("//mediapipe:android", "//mediapipe:ios"): [
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -415,19 +409,13 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util:color_cc_proto",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_quad_renderer",
             "//mediapipe/gpu:shader_util",
         ],
-        "//mediapipe:ios": [
-            "//mediapipe/gpu:gl_calculator_helper",
-            "//mediapipe/gpu:gl_simple_shaders",
-            "//mediapipe/gpu:gpu_buffer",
-            "//mediapipe/gpu:shader_util",
-        ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -486,11 +474,11 @@ cc_library(
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-    ] + selects.with_or({
+    ] + select({
-        ("//mediapipe:android", "//mediapipe:ios"): [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gpu_buffer",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )

mediapipe/calculators/image/bilateral_filter_calculator.cc

@@ -27,11 +27,11 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/shader_util.h"
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace mediapipe {
 
@@ -101,11 +101,11 @@ class BilateralFilterCalculator : public CalculatorBase {
 
   bool use_gpu_ = false;
   bool gpu_initialized_ = false;
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   mediapipe::GlCalculatorHelper gpu_helper_;
   GLuint program_ = 0;
   GLuint program_joint_ = 0;
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(BilateralFilterCalculator);
 
@@ -122,39 +122,46 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
     return ::mediapipe::InternalError("GPU output must have GPU input.");
   }
 
+  bool use_gpu = false;
+
   // Input image to filter.
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
     cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kInputFrameTag)) {
     cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
   }
 
   // Input guide image mask (optional)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag(kInputGuideTagGpu)) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
     cc->Inputs().Tag(kInputGuideTagGpu).Set<mediapipe::GpuBuffer>();
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+    use_gpu |= true;
   }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kInputGuideTag)) {
     cc->Inputs().Tag(kInputGuideTag).Set<ImageFrame>();
   }
 
   // Output image.
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Outputs().HasTag(kOutputFrameTagGpu)) {
     cc->Outputs().Tag(kOutputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Outputs().HasTag(kOutputFrameTag)) {
     cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
   }
 
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+  if (use_gpu) {
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -166,11 +173,11 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
 
   if (cc->Inputs().HasTag(kInputFrameTagGpu) &&
       cc->Outputs().HasTag(kOutputFrameTagGpu)) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     use_gpu_ = true;
 #else
-    RET_CHECK_FAIL() << "GPU processing on non-Android not supported yet.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif
   }
 
   sigma_color_ = options_.sigma_color();
@@ -180,9 +187,9 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
   if (!use_gpu_) sigma_color_ *= 255.0;
 
   if (use_gpu_) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#endif
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
   return ::mediapipe::OkStatus();
@@ -190,33 +197,33 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
 
 ::mediapipe::Status BilateralFilterCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(
+    MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
           if (!gpu_initialized_) {
-            RETURN_IF_ERROR(GlSetup(cc));
+            MP_RETURN_IF_ERROR(GlSetup(cc));
             gpu_initialized_ = true;
           }
-          RETURN_IF_ERROR(RenderGpu(cc));
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
           return ::mediapipe::OkStatus();
         }));
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    RETURN_IF_ERROR(RenderCpu(cc));
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
   }
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status BilateralFilterCalculator::Close(CalculatorContext* cc) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   gpu_helper_.RunInGlContext([this] {
     if (program_) glDeleteProgram(program_);
     program_ = 0;
     if (program_joint_) glDeleteProgram(program_joint_);
     program_joint_ = 0;
   });
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
@@ -263,7 +270,7 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
   if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   const auto& input_frame =
       cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
   auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
@@ -321,13 +328,13 @@ REGISTER_CALCULATOR(BilateralFilterCalculator);
   // Cleanup
   input_texture.Release();
   output_texture.Release();
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -373,11 +380,11 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
   glDeleteVertexArrays(1, &vao);
   glDeleteBuffers(2, vbo);
 
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 }
 
 ::mediapipe::Status BilateralFilterCalculator::GlSetup(CalculatorContext* cc) {
-#if defined(__ANDROID__) || defined(__EMSCRIPTEN__)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   const GLint attr_location[NUM_ATTRIBUTES] = {
       ATTRIB_VERTEX,
       ATTRIB_TEXTURE_POSITION,
@@ -545,7 +552,7 @@ void BilateralFilterCalculator::GlRender(CalculatorContext* cc) {
   glUniform1i(glGetUniformLocation(program_joint_, "input_frame"), 1);
   glUniform1i(glGetUniformLocation(program_joint_, "guide_frame"), 2);
 
-#endif  // __ANDROID__ || __EMSCRIPTEN__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -24,12 +24,12 @@
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/shader_util.h"
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -37,9 +37,20 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 
 namespace mediapipe {
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+namespace {
 
-#endif  // __ANDROID__ or iOS
+#if !defined(MEDIAPIPE_DISABLE_GPU)
+
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
+constexpr char kRectTag[] = "RECT";
+constexpr char kNormRectTag[] = "NORM_RECT";
+constexpr char kHeightTag[] = "HEIGHT";
+constexpr char kImageTag[] = "IMAGE";
+constexpr char kImageGpuTag[] = "IMAGE_GPU";
+constexpr char kWidthTag[] = "WIDTH";
+
+}  // namespace
 
 // Crops the input texture to the given rectangle region. The rectangle can
 // be at arbitrary location on the image with rotation. If there's rotation, the
@@ -91,48 +102,55 @@ class ImageCroppingCalculator : public CalculatorBase {
   bool use_gpu_ = false;
   // Output texture corners (4) after transoformation in normalized coordinates.
   float transformed_points_[8];
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   bool gpu_initialized_ = false;
   mediapipe::GlCalculatorHelper gpu_helper_;
   GLuint program_ = 0;
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(ImageCroppingCalculator);
 
 ::mediapipe::Status ImageCroppingCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Inputs().HasTag(kImageTag) ^ cc->Inputs().HasTag(kImageGpuTag));
-  RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
+  RET_CHECK(cc->Outputs().HasTag(kImageTag) ^
+            cc->Outputs().HasTag(kImageGpuTag));
 
-  if (cc->Inputs().HasTag("IMAGE")) {
+  bool use_gpu = false;
-    RET_CHECK(cc->Outputs().HasTag("IMAGE"));
-    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
-    cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
-  }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
-    RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
-    cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
-    cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
-  }
-#endif  // __ANDROID__ or iOS
 
-  if (cc->Inputs().HasTag("RECT")) {
+  if (cc->Inputs().HasTag(kImageTag)) {
-    cc->Inputs().Tag("RECT").Set<Rect>();
+    RET_CHECK(cc->Outputs().HasTag(kImageTag));
+    cc->Inputs().Tag(kImageTag).Set<ImageFrame>();
+    cc->Outputs().Tag(kImageTag).Set<ImageFrame>();
   }
-  if (cc->Inputs().HasTag("NORM_RECT")) {
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    cc->Inputs().Tag("NORM_RECT").Set<NormalizedRect>();
+  if (cc->Inputs().HasTag(kImageGpuTag)) {
+    RET_CHECK(cc->Outputs().HasTag(kImageGpuTag));
+    cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    use_gpu |= true;
   }
-  if (cc->Inputs().HasTag("WIDTH")) {
+#endif  //  !MEDIAPIPE_DISABLE_GPU
-    cc->Inputs().Tag("WIDTH").Set<int>();
+
+  RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ cc->Inputs().HasTag(kNormRectTag));
+  if (cc->Inputs().HasTag(kRectTag)) {
+    cc->Inputs().Tag(kRectTag).Set<Rect>();
   }
-  if (cc->Inputs().HasTag("HEIGHT")) {
+  if (cc->Inputs().HasTag(kNormRectTag)) {
-    cc->Inputs().Tag("HEIGHT").Set<int>();
+    cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>();
+  }
+  if (cc->Inputs().HasTag(kWidthTag)) {
+    cc->Inputs().Tag(kWidthTag).Set<int>();
+  }
+  if (cc->Inputs().HasTag(kHeightTag)) {
+    cc->Inputs().Tag(kHeightTag).Set<int>();
   }
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+  if (use_gpu) {
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-#endif  // __ANDROID__ or iOS
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -140,56 +158,68 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
 ::mediapipe::Status ImageCroppingCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  if (cc->Inputs().HasTag("IMAGE_GPU")) {
+  if (cc->Inputs().HasTag(kImageGpuTag)) {
     use_gpu_ = true;
   }
 
   options_ = cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
 
   if (use_gpu_) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #else
     RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::Process(CalculatorContext* cc) {
+  if (cc->Inputs().HasTag(kRectTag) && cc->Inputs().Tag(kRectTag).IsEmpty()) {
+    VLOG(1) << "RECT is empty for timestamp: " << cc->InputTimestamp();
+    return ::mediapipe::OkStatus();
+  }
+  if (cc->Inputs().HasTag(kNormRectTag) &&
+      cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
+    VLOG(1) << "NORM_RECT is empty for timestamp: " << cc->InputTimestamp();
+    return ::mediapipe::OkStatus();
+  }
   if (use_gpu_) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(
+    MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
           if (!gpu_initialized_) {
-            RETURN_IF_ERROR(InitGpu(cc));
+            MP_RETURN_IF_ERROR(InitGpu(cc));
             gpu_initialized_ = true;
           }
-          RETURN_IF_ERROR(RenderGpu(cc));
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
           return ::mediapipe::OkStatus();
         }));
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    RETURN_IF_ERROR(RenderCpu(cc));
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
   }
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::Close(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   gpu_helper_.RunInGlContext([this] {
     if (program_) glDeleteProgram(program_);
     program_ = 0;
   });
   gpu_initialized_ = false;
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
-  const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
+  if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
+    return ::mediapipe::OkStatus();
+  }
+  const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
   float rect_center_x = input_img.Width() / 2.0f;
@@ -197,8 +227,8 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   float rotation = 0.0f;
   int target_width = input_img.Width();
   int target_height = input_img.Height();
-  if (cc->Inputs().HasTag("RECT")) {
+  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag("RECT").Get<Rect>();
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
         rect.y_center() >= 0) {
       rect_center_x = rect.x_center();
@@ -207,8 +237,8 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
       target_height = rect.height();
       rotation = rect.rotation();
     }
-  } else if (cc->Inputs().HasTag("NORM_RECT")) {
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag("NORM_RECT").Get<NormalizedRect>();
+    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
     if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
         rect.y_center() >= 0.0) {
       rect_center_x = std::round(rect.x_center() * input_img.Width());
@@ -218,9 +248,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
       rotation = rect.rotation();
     }
   } else {
-    if (cc->Inputs().HasTag("WIDTH") && cc->Inputs().HasTag("HEIGHT")) {
+    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      target_width = cc->Inputs().Tag("WIDTH").Get<int>();
+      target_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      target_height = cc->Inputs().Tag("HEIGHT").Get<int>();
+      target_height = cc->Inputs().Tag(kHeightTag).Get<int>();
     } else if (options_.has_width() && options_.has_height()) {
       target_width = options_.width();
       target_height = options_.height();
@@ -253,16 +283,17 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
       input_img.Format(), cropped_image.cols, cropped_image.rows));
   cv::Mat output_mat = formats::MatView(output_frame.get());
   cropped_image.copyTo(output_mat);
-  cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kImageTag).Add(output_frame.release(),
+                                   cc->InputTimestamp());
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) {
-  if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+  if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-  const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
+  const Packet& input_packet = cc->Inputs().Tag(kImageGpuTag).Value();
   const auto& input_buffer = input_packet.Get<mediapipe::GpuBuffer>();
   auto src_tex = gpu_helper_.CreateSourceTexture(input_buffer);
 
@@ -287,18 +318,18 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
 
   // Send result image in GPU packet.
   auto output = dst_tex.GetFrame<mediapipe::GpuBuffer>();
-  cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
+  cc->Outputs().Tag(kImageGpuTag).Add(output.release(), cc->InputTimestamp());
 
   // Cleanup
   src_tex.Release();
   dst_tex.Release();
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 void ImageCroppingCalculator::GlRender() {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -342,11 +373,11 @@ void ImageCroppingCalculator::GlRender() {
   glDeleteVertexArrays(1, &vao);
   glDeleteBuffers(2, vbo);
 
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 }
 
 ::mediapipe::Status ImageCroppingCalculator::InitGpu(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   const GLint attr_location[NUM_ATTRIBUTES] = {
       ATTRIB_VERTEX,
       ATTRIB_TEXTURE_POSITION,
@@ -392,7 +423,7 @@ void ImageCroppingCalculator::GlRender() {
   // Parameters
   glUseProgram(program_);
   glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
@@ -410,8 +441,8 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
   int y_center = src_height / 2;
   // Get the rotation of the cropping box.
   float rotation = 0.0f;
-  if (cc->Inputs().HasTag("RECT")) {
+  if (cc->Inputs().HasTag(kRectTag)) {
-    const auto& rect = cc->Inputs().Tag("RECT").Get<Rect>();
+    const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     // Only use the rect if it is valid.
     if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
         rect.y_center() >= 0) {
@@ -421,8 +452,8 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
       crop_height = rect.height();
       rotation = rect.rotation();
     }
-  } else if (cc->Inputs().HasTag("NORM_RECT")) {
+  } else if (cc->Inputs().HasTag(kNormRectTag)) {
-    const auto& rect = cc->Inputs().Tag("NORM_RECT").Get<NormalizedRect>();
+    const auto& rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
     // Only use the rect if it is valid.
     if (rect.width() > 0.0 && rect.height() > 0.0 && rect.x_center() >= 0.0 &&
         rect.y_center() >= 0.0) {
@@ -433,9 +464,9 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
       rotation = rect.rotation();
     }
   } else {
-    if (cc->Inputs().HasTag("WIDTH") && cc->Inputs().HasTag("HEIGHT")) {
+    if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) {
-      crop_width = cc->Inputs().Tag("WIDTH").Get<int>();
+      crop_width = cc->Inputs().Tag(kWidthTag).Get<int>();
-      crop_height = cc->Inputs().Tag("HEIGHT").Get<int>();
+      crop_height = cc->Inputs().Tag(kHeightTag).Get<int>();
     } else if (options_.has_width() && options_.has_height()) {
       crop_width = options_.width();
       crop_height = options_.height();

mediapipe/calculators/image/image_properties_calculator.cc

@@ -15,9 +15,9 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gpu_buffer.h"
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace mediapipe {
 
@@ -44,11 +44,11 @@ class ImagePropertiesCalculator : public CalculatorBase {
     if (cc->Inputs().HasTag("IMAGE")) {
       cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
     }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     if (cc->Inputs().HasTag("IMAGE_GPU")) {
       cc->Inputs().Tag("IMAGE_GPU").Set<::mediapipe::GpuBuffer>();
     }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
     if (cc->Outputs().HasTag("SIZE")) {
       cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
@@ -71,7 +71,7 @@ class ImagePropertiesCalculator : public CalculatorBase {
       width = image.Width();
       height = image.Height();
     }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     if (cc->Inputs().HasTag("IMAGE_GPU") &&
         !cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
       const auto& image =
@@ -79,7 +79,7 @@ class ImagePropertiesCalculator : public CalculatorBase {
       width = image.width();
       height = image.height();
     }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
     cc->Outputs().Tag("SIZE").AddPacket(
         MakePacket<std::pair<int, int>>(width, height)

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -22,12 +22,12 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/gpu/scale_mode.pb.h"
 
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_quad_renderer.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
 #include "mediapipe/gpu/shader_util.h"
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 #if defined(__ANDROID__)
 // The size of Java arrays is dynamic, which makes it difficult to
@@ -42,9 +42,9 @@ typedef int DimensionsPacketType[2];
 
 namespace mediapipe {
 
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
 int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
@@ -170,12 +170,12 @@ class ImageTransformationCalculator : public CalculatorBase {
   mediapipe::ScaleMode_Mode scale_mode_;
 
   bool use_gpu_ = false;
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   GlCalculatorHelper helper_;
   std::unique_ptr<QuadRenderer> rgb_renderer_;
   std::unique_ptr<QuadRenderer> yuv_renderer_;
   std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(ImageTransformationCalculator);
 
@@ -185,18 +185,22 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
   RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
 
+  bool use_gpu = false;
+
   if (cc->Inputs().HasTag("IMAGE")) {
     RET_CHECK(cc->Outputs().HasTag("IMAGE"));
     cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
     cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
   }
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag("IMAGE_GPU")) {
     RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
     cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
     cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
   if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
     cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
   }
@@ -212,9 +216,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
   }
 
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+  if (use_gpu) {
-  RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-#endif  // __ANDROID__ || iOS
+    MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -244,18 +250,18 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     rotation_ = DegreesToRotationMode(
         cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
   } else {
-    rotation_ = DegreesToRotationMode(options_.rotation_mode());
+    rotation_ = options_.rotation_mode();
   }
 
   scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
 
   if (use_gpu_) {
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     // Let the helper access the GL context information.
-    RETURN_IF_ERROR(helper_.Open(cc));
+    MP_RETURN_IF_ERROR(helper_.Open(cc));
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
   return ::mediapipe::OkStatus();
@@ -264,10 +270,10 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 ::mediapipe::Status ImageTransformationCalculator::Process(
     CalculatorContext* cc) {
   if (use_gpu_) {
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     return helper_.RunInGlContext(
         [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
     return RenderCpu(cc);
   }
@@ -277,7 +283,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 ::mediapipe::Status ImageTransformationCalculator::Close(
     CalculatorContext* cc) {
   if (use_gpu_) {
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     QuadRenderer* rgb_renderer = rgb_renderer_.release();
     QuadRenderer* yuv_renderer = yuv_renderer_.release();
     QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release();
@@ -295,8 +301,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         delete yuv_renderer;
       }
     });
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -371,7 +378,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
 ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
     CalculatorContext* cc) {
-#if defined(__ANDROID__) || defined(__APPLE__) && !TARGET_OS_OSX
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   int input_width = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().width();
   int input_height = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>().height();
 
@@ -398,7 +405,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
       input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) {
     if (!yuv_renderer_) {
       yuv_renderer_ = absl::make_unique<QuadRenderer>();
-      RETURN_IF_ERROR(
+      MP_RETURN_IF_ERROR(
           yuv_renderer_->GlSetup(::mediapipe::kYUV2TexToRGBFragmentShader,
                                  {"video_frame_y", "video_frame_uv"}));
     }
@@ -408,20 +415,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 #endif    // iOS
   {
     src1 = helper_.CreateSourceTexture(input);
-#if defined(__ANDROID__)
+#if defined(TEXTURE_EXTERNAL_OES)
     if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
       if (!ext_rgb_renderer_) {
         ext_rgb_renderer_ = absl::make_unique<QuadRenderer>();
-        RETURN_IF_ERROR(ext_rgb_renderer_->GlSetup(
+        MP_RETURN_IF_ERROR(ext_rgb_renderer_->GlSetup(
             ::mediapipe::kBasicTexturedFragmentShaderOES, {"video_frame"}));
       }
       renderer = ext_rgb_renderer_.get();
     } else  // NOLINT(readability/braces)
-#endif      // __ANDROID__
+#endif      // TEXTURE_EXTERNAL_OES
     {
       if (!rgb_renderer_) {
         rgb_renderer_ = absl::make_unique<QuadRenderer>();
-        RETURN_IF_ERROR(rgb_renderer_->GlSetup());
+        MP_RETURN_IF_ERROR(rgb_renderer_->GlSetup());
       }
       renderer = rgb_renderer_.get();
     }
@@ -446,7 +453,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   glActiveTexture(GL_TEXTURE1);
   glBindTexture(src1.target(), src1.name());
 
-  RETURN_IF_ERROR(renderer->GlRender(
+  MP_RETURN_IF_ERROR(renderer->GlRender(
       src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
       rotation, options_.flip_horizontally(), options_.flip_vertically(),
       /*flip_texture=*/false));
@@ -460,7 +467,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   auto output = dst.GetFrame<GpuBuffer>();
   cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
 
-#endif  // __ANDROID__ || iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/image/mask_overlay_calculator.cc

@@ -74,7 +74,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
 
 // static
 ::mediapipe::Status MaskOverlayCalculator::GetContract(CalculatorContract* cc) {
-  RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
+  MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
   cc->Inputs().Get("VIDEO", 0).Set<GpuBuffer>();
   cc->Inputs().Get("VIDEO", 1).Set<GpuBuffer>();
   if (cc->Inputs().HasTag("MASK"))
@@ -103,7 +103,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
       const auto& options = cc->Options<MaskOverlayCalculatorOptions>();
       const auto mask_channel = options.mask_channel();
 
-      RETURN_IF_ERROR(GlSetup(mask_channel));
+      MP_RETURN_IF_ERROR(GlSetup(mask_channel));
       initialized_ = true;
     }
 
@@ -147,7 +147,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
       glActiveTexture(GL_TEXTURE3);
       glBindTexture(mask_tex.target(), mask_tex.name());
 
-      RETURN_IF_ERROR(GlRender(mask_const));
+      MP_RETURN_IF_ERROR(GlRender(mask_const));
 
       glActiveTexture(GL_TEXTURE3);
       glBindTexture(mask_tex.target(), 0);
@@ -155,7 +155,7 @@ REGISTER_CALCULATOR(MaskOverlayCalculator);
     } else {
       const float mask_const = mask_packet.Get<float>();
 
-      RETURN_IF_ERROR(GlRender(mask_const));
+      MP_RETURN_IF_ERROR(GlRender(mask_const));
     }
 
     glActiveTexture(GL_TEXTURE2);

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc

@@ -30,7 +30,7 @@ namespace {
 
 TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
   std::string contents;
-  MEDIAPIPE_ASSERT_OK(file::GetContents(
+  MP_ASSERT_OK(file::GetContents(
       file::JoinPath("./", "/mediapipe/calculators/image/testdata/dino.jpg"),
       &contents));
   Packet input_packet = MakePacket<std::string>(contents);
@@ -44,7 +44,7 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) {
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const auto& outputs = runner.Outputs();
   ASSERT_EQ(1, outputs.NumEntries());
   const std::vector<Packet>& packets = outputs.Index(0).packets;
@@ -87,7 +87,7 @@ TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) {
   CalculatorRunner runner(node_config);
   runner.MutableInputs()->Index(0).packets.push_back(
       input_packet.At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const auto& outputs = runner.Outputs();
   ASSERT_EQ(1, outputs.NumEntries());
   const std::vector<Packet>& packets = outputs.Index(0).packets;

mediapipe/calculators/image/opencv_image_encoder_calculator_test.cc

@@ -55,7 +55,7 @@ TEST(OpenCvImageEncoderCalculatorTest, TestJpegWithQualities) {
     CalculatorRunner runner(node_config);
     runner.MutableInputs()->Index(0).packets.push_back(
         input_packet.At(Timestamp(0)));
-    MEDIAPIPE_ASSERT_OK(runner.Run());
+    MP_ASSERT_OK(runner.Run());
     const auto& outputs = runner.Outputs();
     ASSERT_EQ(1, outputs.NumEntries());
     const std::vector<Packet>& packets = outputs.Index(0).packets;

mediapipe/calculators/image/recolor_calculator.cc

@@ -21,12 +21,11 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/util/color.pb.h"
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
-#include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/shader_util.h"
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace {
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -95,10 +94,10 @@ class RecolorCalculator : public CalculatorBase {
   mediapipe::RecolorCalculatorOptions::MaskChannel mask_channel_;
 
   bool use_gpu_ = false;
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   mediapipe::GlCalculatorHelper gpu_helper_;
   GLuint program_ = 0;
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(RecolorCalculator);
 
@@ -107,36 +106,43 @@ REGISTER_CALCULATOR(RecolorCalculator);
   RET_CHECK(!cc->Inputs().GetTags().empty());
   RET_CHECK(!cc->Outputs().GetTags().empty());
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+  bool use_gpu = false;
+
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag("IMAGE_GPU")) {
     cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag("IMAGE")) {
     cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
   }
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag("MASK_GPU")) {
     cc->Inputs().Tag("MASK_GPU").Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag("MASK")) {
     cc->Inputs().Tag("MASK").Set<ImageFrame>();
   }
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Outputs().HasTag("IMAGE_GPU")) {
     cc->Outputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Outputs().HasTag("IMAGE")) {
     cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
   }
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+  if (use_gpu) {
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-#endif  // __ANDROID__ or iOS
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -146,42 +152,42 @@ REGISTER_CALCULATOR(RecolorCalculator);
 
   if (cc->Inputs().HasTag("IMAGE_GPU")) {
     use_gpu_ = true;
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  RETURN_IF_ERROR(LoadOptions(cc));
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status RecolorCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(
+    MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
           if (!initialized_) {
-            RETURN_IF_ERROR(InitGpu(cc));
+            MP_RETURN_IF_ERROR(InitGpu(cc));
             initialized_ = true;
           }
-          RETURN_IF_ERROR(RenderGpu(cc));
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
           return ::mediapipe::OkStatus();
         }));
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    RETURN_IF_ERROR(RenderCpu(cc));
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
   }
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status RecolorCalculator::Close(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   gpu_helper_.RunInGlContext([this] {
     if (program_) glDeleteProgram(program_);
     program_ = 0;
   });
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
@@ -194,7 +200,7 @@ REGISTER_CALCULATOR(RecolorCalculator);
   if (cc->Inputs().Tag("MASK_GPU").IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   // Get inputs and setup output.
   const Packet& input_packet = cc->Inputs().Tag("IMAGE_GPU").Value();
   const Packet& mask_packet = cc->Inputs().Tag("MASK_GPU").Value();
@@ -233,13 +239,13 @@ REGISTER_CALCULATOR(RecolorCalculator);
   img_tex.Release();
   mask_tex.Release();
   dst_tex.Release();
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 void RecolorCalculator::GlRender() {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -287,7 +293,7 @@ void RecolorCalculator::GlRender() {
   glBindVertexArray(0);
   glDeleteVertexArrays(1, &vao);
   glDeleteBuffers(2, vbo);
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 }
 
 ::mediapipe::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) {
@@ -305,7 +311,7 @@ void RecolorCalculator::GlRender() {
 }
 
 ::mediapipe::Status RecolorCalculator::InitGpu(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   const GLint attr_location[NUM_ATTRIBUTES] = {
       ATTRIB_VERTEX,
       ATTRIB_TEXTURE_POSITION,
@@ -374,7 +380,7 @@ void RecolorCalculator::GlRender() {
   glUniform1i(glGetUniformLocation(program_, "mask"), 2);
   glUniform3f(glGetUniformLocation(program_, "recolor"), color_[0], color_[1],
               color_[2]);
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/image/scale_image_calculator.cc

@@ -253,21 +253,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
 
 ::mediapipe::Status ScaleImageCalculator::InitializeFrameInfo(
     CalculatorContext* cc) {
-  RETURN_IF_ERROR(
+  MP_RETURN_IF_ERROR(
       scale_image::FindCropDimensions(input_width_, input_height_,  //
                                       options_.min_aspect_ratio(),  //
                                       options_.max_aspect_ratio(),  //
                                       &crop_width_, &crop_height_,  //
                                       &col_start_, &row_start_));
-  RETURN_IF_ERROR(
+  MP_RETURN_IF_ERROR(
       scale_image::FindOutputDimensions(crop_width_, crop_height_,            //
                                         options_.target_width(),              //
                                         options_.target_height(),             //
                                         options_.preserve_aspect_ratio(),     //
                                         options_.scale_to_multiple_of_two(),  //
                                         &output_width_, &output_height_));
-  RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
+  MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
-                                             &interpolation_algorithm_));
+                                                &interpolation_algorithm_));
   if (interpolation_algorithm_ == -1 &&
       (output_width_ > crop_width_ || output_height_ > crop_height_)) {
     output_width_ = crop_width_;
@@ -327,7 +327,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
   bool has_override_options = cc->Inputs().HasTag("OVERRIDE_OPTIONS");
 
   if (!has_override_options) {
-    RETURN_IF_ERROR(InitializeFromOptions());
+    MP_RETURN_IF_ERROR(InitializeFromOptions());
   }
 
   if (!cc->Inputs().Get(input_data_id_).Header().IsEmpty()) {
@@ -377,8 +377,8 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
     if (input_width_ > 0 && input_height_ > 0 &&
         input_format_ != ImageFormat::UNKNOWN &&
         output_format_ != ImageFormat::UNKNOWN) {
-      RETURN_IF_ERROR(ValidateImageFormats());
+      MP_RETURN_IF_ERROR(ValidateImageFormats());
-      RETURN_IF_ERROR(InitializeFrameInfo(cc));
+      MP_RETURN_IF_ERROR(InitializeFrameInfo(cc));
       std::unique_ptr<VideoHeader> output_header(new VideoHeader());
       *output_header = input_video_header_;
       output_header->format = output_format_;
@@ -461,9 +461,9 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
       } else {
         output_format_ = input_format_;
       }
-      RETURN_IF_ERROR(InitializeFrameInfo(cc));
+      MP_RETURN_IF_ERROR(InitializeFrameInfo(cc));
     }
-    RETURN_IF_ERROR(ValidateImageFormats());
+    MP_RETURN_IF_ERROR(ValidateImageFormats());
   } else {
     if (input_width_ != image_frame.Width() ||
         input_height_ != image_frame.Height()) {
@@ -503,9 +503,9 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
       } else {
         output_format_ = input_format_;
       }
-      RETURN_IF_ERROR(InitializeFrameInfo(cc));
+      MP_RETURN_IF_ERROR(InitializeFrameInfo(cc));
     }
-    RETURN_IF_ERROR(ValidateImageFormats());
+    MP_RETURN_IF_ERROR(ValidateImageFormats());
   } else {
     if (input_width_ != yuv_image.width() ||
         input_height_ != yuv_image.height()) {
@@ -531,7 +531,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
       options_.MergeFrom(cc->Inputs()
                              .Tag("OVERRIDE_OPTIONS")
                              .Get<ScaleImageCalculatorOptions>());
-      RETURN_IF_ERROR(InitializeFromOptions());
+      MP_RETURN_IF_ERROR(InitializeFromOptions());
     }
     if (cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") &&
         !cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) {
@@ -548,7 +548,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
   if (input_format_ == ImageFormat::YCBCR420P) {
     const YUVImage* yuv_image =
         &cc->Inputs().Get(input_data_id_).Get<YUVImage>();
-    RETURN_IF_ERROR(ValidateYUVImage(cc, *yuv_image));
+    MP_RETURN_IF_ERROR(ValidateYUVImage(cc, *yuv_image));
 
     if (output_format_ == ImageFormat::SRGB) {
       // TODO: For ease of implementation, YUVImage is converted to
@@ -596,7 +596,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
     }
   } else {
     image_frame = &cc->Inputs().Get(input_data_id_).Get<ImageFrame>();
-    RETURN_IF_ERROR(ValidateImageFrame(cc, *image_frame));
+    MP_RETURN_IF_ERROR(ValidateImageFrame(cc, *image_frame));
   }
 
   std::unique_ptr<ImageFrame> cropped_image;

mediapipe/calculators/image/scale_image_utils_test.cc

@@ -28,8 +28,8 @@ TEST(ScaleImageUtilsTest, FindCropDimensions) {
   int col_start;
   int row_start;
   // No cropping because aspect ratios should be ignored.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(50, 100, "0/1", "1/0", &crop_width,
+  MP_ASSERT_OK(FindCropDimensions(50, 100, "0/1", "1/0", &crop_width,
-                                         &crop_height, &col_start, &row_start));
+                                  &crop_height, &col_start, &row_start));
   EXPECT_EQ(50, crop_width);
   EXPECT_EQ(100, crop_height);
   EXPECT_EQ(0, row_start);
@@ -37,39 +37,38 @@ TEST(ScaleImageUtilsTest, FindCropDimensions) {
 
   // Tests proto examples.
   // 16:9 aspect ratio, should be unchanged.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(1920, 1080, "9/16", "16/9",
+  MP_ASSERT_OK(FindCropDimensions(1920, 1080, "9/16", "16/9", &crop_width,
-                                         &crop_width, &crop_height, &col_start,
+                                  &crop_height, &col_start, &row_start));
-                                         &row_start));
   EXPECT_EQ(0, col_start);
   EXPECT_EQ(1920, crop_width);
   EXPECT_EQ(0, row_start);
   EXPECT_EQ(1080, crop_height);
   // 10:16 aspect ratio, should be unchanged.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(640, 1024, "9/16", "16/9", &crop_width,
+  MP_ASSERT_OK(FindCropDimensions(640, 1024, "9/16", "16/9", &crop_width,
-                                         &crop_height, &col_start, &row_start));
+                                  &crop_height, &col_start, &row_start));
   EXPECT_EQ(0, col_start);
   EXPECT_EQ(640, crop_width);
   EXPECT_EQ(0, row_start);
   EXPECT_EQ(1024, crop_height);
 
   // 2:1 aspect ratio, width is cropped.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(640, 320, "9/16", "16/9", &crop_width,
+  MP_ASSERT_OK(FindCropDimensions(640, 320, "9/16", "16/9", &crop_width,
-                                         &crop_height, &col_start, &row_start));
+                                  &crop_height, &col_start, &row_start));
   EXPECT_EQ(36, col_start);
   EXPECT_EQ(568, crop_width);
   EXPECT_EQ(0, row_start);
   EXPECT_EQ(320, crop_height);
   // 1:5 aspect ratio, height is cropped.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(96, 480, "9/16", "16/9", &crop_width,
+  MP_ASSERT_OK(FindCropDimensions(96, 480, "9/16", "16/9", &crop_width,
-                                         &crop_height, &col_start, &row_start));
+                                  &crop_height, &col_start, &row_start));
   EXPECT_EQ(0, col_start);
   EXPECT_EQ(96, crop_width);
   EXPECT_EQ(155, row_start);
   EXPECT_EQ(170, crop_height);
 
   // Tests min = max, crops width.
-  MEDIAPIPE_ASSERT_OK(FindCropDimensions(200, 100, "1/1", "1/1", &crop_width,
+  MP_ASSERT_OK(FindCropDimensions(200, 100, "1/1", "1/1", &crop_width,
-                                         &crop_height, &col_start, &row_start));
+                                  &crop_height, &col_start, &row_start));
   EXPECT_EQ(50, col_start);
   EXPECT_EQ(100, crop_width);
   EXPECT_EQ(0, row_start);
@@ -80,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
   int output_width;
   int output_height;
   // Not scale.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
-                                           &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(100, output_height);
   // Not scale with odd input size.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
+  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(201, output_width);
   EXPECT_EQ(101, output_height);
   // Scale down by 1/2.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale up, doubling dimensions.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(400, output_width);
   EXPECT_EQ(200, output_height);
   // Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
   // visible by 2.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(74, output_height);
   // Fits a 2:1 image into a 400 x 50 box.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to odd size.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(151, output_width);
   EXPECT_EQ(101, output_height);
 }
@@ -132,18 +131,18 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
   int output_width;
   int output_height;
   // Scale width only.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(100, output_height);
   // Scale height only.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(200, output_height);
   // Scale both dimensions.
-  MEDIAPIPE_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
-                                           &output_width, &output_height));
+                                    &output_width, &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(200, output_height);
 }

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -25,12 +25,11 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
-#include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/shader_util.h"
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 namespace mediapipe {
 
@@ -107,16 +106,18 @@ class SetAlphaCalculator : public CalculatorBase {
 
   bool use_gpu_ = false;
   bool gpu_initialized_ = false;
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   mediapipe::GlCalculatorHelper gpu_helper_;
   GLuint program_ = 0;
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 };
 REGISTER_CALCULATOR(SetAlphaCalculator);
 
 ::mediapipe::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
   CHECK_GE(cc->Inputs().NumEntries(), 1);
 
+  bool use_gpu = false;
+
   if (cc->Inputs().HasTag(kInputFrameTag) &&
       cc->Inputs().HasTag(kInputFrameTagGpu)) {
     return ::mediapipe::InternalError("Cannot have multiple input images.");
@@ -127,38 +128,43 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
   }
 
   // Input image to add/edit alpha channel.
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
     cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kInputFrameTag)) {
     cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
   }
 
   // Input alpha image mask (optional)
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag(kInputAlphaTagGpu)) {
     cc->Inputs().Tag(kInputAlphaTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kInputAlphaTag)) {
     cc->Inputs().Tag(kInputAlphaTag).Set<ImageFrame>();
   }
 
   // RGBA output image.
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Outputs().HasTag(kOutputFrameTagGpu)) {
     cc->Outputs().Tag(kOutputFrameTagGpu).Set<mediapipe::GpuBuffer>();
+    use_gpu |= true;
   }
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Outputs().HasTag(kOutputFrameTag)) {
     cc->Outputs().Tag(kOutputFrameTag).Set<ImageFrame>();
   }
 
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+  if (use_gpu) {
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-#endif  // __ANDROID__ or iOS
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -170,11 +176,11 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
 
   if (cc->Inputs().HasTag(kInputFrameTagGpu) &&
       cc->Outputs().HasTag(kOutputFrameTagGpu)) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
     use_gpu_ = true;
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
   // Get global value from options (-1 if not set).
@@ -187,41 +193,41 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
     RET_CHECK_FAIL() << "Must use either image mask or options alpha value.";
 
   if (use_gpu_) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #endif
-  }
+  }  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status SetAlphaCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
-    RETURN_IF_ERROR(
+    MP_RETURN_IF_ERROR(
         gpu_helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
           if (!gpu_initialized_) {
-            RETURN_IF_ERROR(GlSetup(cc));
+            MP_RETURN_IF_ERROR(GlSetup(cc));
             gpu_initialized_ = true;
           }
-          RETURN_IF_ERROR(RenderGpu(cc));
+          MP_RETURN_IF_ERROR(RenderGpu(cc));
           return ::mediapipe::OkStatus();
         }));
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
-    RETURN_IF_ERROR(RenderCpu(cc));
+    MP_RETURN_IF_ERROR(RenderCpu(cc));
   }
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status SetAlphaCalculator::Close(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   gpu_helper_.RunInGlContext([this] {
     if (program_) glDeleteProgram(program_);
     program_ = 0;
   });
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
@@ -295,7 +301,7 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
   if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   // Setup source texture.
   const auto& input_frame =
       cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
@@ -348,13 +354,13 @@ REGISTER_CALCULATOR(SetAlphaCalculator);
   // Cleanup
   input_texture.Release();
   output_texture.Release();
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }
 
 void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   static const GLfloat square_vertices[] = {
       -1.0f, -1.0f,  // bottom left
       1.0f,  -1.0f,  // bottom right
@@ -403,11 +409,11 @@ void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
   glDeleteVertexArrays(1, &vao);
   glDeleteBuffers(2, vbo);
 
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 }
 
 ::mediapipe::Status SetAlphaCalculator::GlSetup(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU)
   const GLint attr_location[NUM_ATTRIBUTES] = {
       ATTRIB_VERTEX,
       ATTRIB_TEXTURE_POSITION,
@@ -460,7 +466,7 @@ void SetAlphaCalculator::GlRender(CalculatorContext* cc) {
   glUniform1i(glGetUniformLocation(program_, "alpha_mask"), 2);
   glUniform1f(glGetUniformLocation(program_, "alpha_value"), alpha_value_);
 
-#endif  // __ANDROID__ or iOS
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/internal/BUILD

@@ -29,7 +29,7 @@ mediapipe_cc_proto_library(
     name = "callback_packet_calculator_cc_proto",
     srcs = ["callback_packet_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe/framework:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":callback_packet_calculator_proto"],
 )
 

mediapipe/calculators/tensorflow/BUILD

@@ -22,7 +22,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
 proto_library(
     name = "graph_tensors_packet_generator_proto",
     srcs = ["graph_tensors_packet_generator.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework:packet_generator_proto",
@@ -104,6 +104,17 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "unpack_media_sequence_calculator_proto",
+    srcs = ["unpack_media_sequence_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/core:packet_resampler_calculator_proto",
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util:audio_decoder_proto",
+    ],
+)
+
 proto_library(
     name = "vector_float_to_tensor_calculator_options_proto",
     srcs = ["vector_float_to_tensor_calculator_options.proto"],
@@ -118,7 +129,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework:packet_generator_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":graph_tensors_packet_generator_proto"],
 )
 
@@ -129,7 +140,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":image_frame_to_tensor_calculator_proto"],
 )
 
@@ -137,7 +148,7 @@ mediapipe_cc_proto_library(
     name = "matrix_to_tensor_calculator_options_cc_proto",
     srcs = ["matrix_to_tensor_calculator_options.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":matrix_to_tensor_calculator_options_proto"],
 )
 
@@ -145,7 +156,7 @@ mediapipe_cc_proto_library(
     name = "lapped_tensor_buffer_calculator_cc_proto",
     srcs = ["lapped_tensor_buffer_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":lapped_tensor_buffer_calculator_proto"],
 )
 
@@ -153,7 +164,7 @@ mediapipe_cc_proto_library(
     name = "object_detection_tensors_to_detections_calculator_cc_proto",
     srcs = ["object_detection_tensors_to_detections_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":object_detection_tensors_to_detections_calculator_proto"],
 )
 
@@ -164,7 +175,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":pack_media_sequence_calculator_proto"],
 )
 
@@ -172,7 +183,7 @@ mediapipe_cc_proto_library(
     name = "tensorflow_inference_calculator_cc_proto",
     srcs = ["tensorflow_inference_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensorflow_inference_calculator_proto"],
 )
 
@@ -183,15 +194,26 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:packet_generator_cc_proto",
         "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tensorflow_session_from_frozen_graph_calculator_cc_proto",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "tensorflow_session_from_saved_model_generator_cc_proto",
     srcs = ["tensorflow_session_from_saved_model_generator.proto"],
     cc_deps = ["//mediapipe/framework:packet_generator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_saved_model_generator_proto"],
 )
 
@@ -199,7 +221,7 @@ mediapipe_cc_proto_library(
     name = "tensorflow_session_from_saved_model_calculator_cc_proto",
     srcs = ["tensorflow_session_from_saved_model_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_saved_model_calculator_proto"],
 )
 
@@ -207,7 +229,7 @@ mediapipe_cc_proto_library(
     name = "tensor_squeeze_dimensions_calculator_cc_proto",
     srcs = ["tensor_squeeze_dimensions_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensor_squeeze_dimensions_calculator_proto"],
 )
 
@@ -215,7 +237,7 @@ mediapipe_cc_proto_library(
     name = "tensor_to_image_frame_calculator_cc_proto",
     srcs = ["tensor_to_image_frame_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensor_to_image_frame_calculator_proto"],
 )
 
@@ -226,7 +248,7 @@ mediapipe_cc_proto_library(
         "//mediapipe/framework:calculator_cc_proto",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensor_to_matrix_calculator_proto"],
 )
 
@@ -234,7 +256,7 @@ mediapipe_cc_proto_library(
     name = "tensor_to_vector_float_calculator_options_cc_proto",
     srcs = ["tensor_to_vector_float_calculator_options.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tensor_to_vector_float_calculator_options_proto"],
 )
 
@@ -244,16 +266,28 @@ mediapipe_cc_proto_library(
     cc_deps = [
         "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
         "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util:audio_decoder_cc_proto",
     ],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":unpack_media_sequence_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "vector_int_to_tensor_calculator_options_cc_proto",
+    srcs = ["vector_int_to_tensor_calculator_options.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":vector_int_to_tensor_calculator_options_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "vector_float_to_tensor_calculator_options_cc_proto",
     srcs = ["vector_float_to_tensor_calculator_options.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":vector_float_to_tensor_calculator_options_proto"],
 )
 
@@ -444,6 +478,35 @@ cc_library(
     }),
 )
 
+cc_library(
+    name = "tensorflow_session_from_frozen_graph_calculator",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator.cc"],
+    features = ["no_layering_check"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tensorflow_session",
+        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/tool:status_util",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:ret_check",
+    ] + select({
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+            "@org_tensorflow//tensorflow/core:core",
+        ],
+        "//mediapipe:android": [
+            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "//mediapipe/android/file/base",
+        ],
+        "//mediapipe:ios": [
+            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "//mediapipe/android/file/base",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tensorflow_session_from_frozen_graph_generator",
     srcs = ["tensorflow_session_from_frozen_graph_generator.cc"],
@@ -580,6 +643,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tfrecord_reader_calculator",
+    srcs = ["tfrecord_reader_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/core:lib",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tensor_to_vector_float_calculator",
     srcs = ["tensor_to_vector_float_calculator.cc"],
@@ -613,6 +692,7 @@ cc_library(
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/util:audio_decoder_cc_proto",
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:protos_all_cc",
@@ -620,6 +700,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "vector_int_to_tensor_calculator",
+    srcs = ["vector_int_to_tensor_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":vector_int_to_tensor_calculator_options_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/core:framework",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "vector_float_to_tensor_calculator",
     srcs = ["vector_float_to_tensor_calculator.cc"],
@@ -634,6 +728,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "unpack_yt8m_sequence_example_calculator",
+    srcs = ["unpack_yt8m_sequence_example_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "graph_tensors_packet_generator_test",
     srcs = ["graph_tensors_packet_generator_test.cc"],
@@ -729,7 +837,6 @@ cc_test(
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:opencv_imgcodecs",
-        "//mediapipe/framework/port:status",
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -737,6 +844,36 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "tensorflow_session_from_frozen_graph_calculator_test",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator_test.cc"],
+    data = [":test_frozen_graph"],
+    linkstatic = 1,
+    deps = [
+        ":tensorflow_inference_calculator",
+        ":tensorflow_session",
+        ":tensorflow_session_from_frozen_graph_calculator",
+        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:tag_map_helper",
+        "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/strings",
+        "@org_tensorflow//tensorflow/core:direct_session",
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:testlib",
+        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
+        "@org_tensorflow//tensorflow/core/kernels:math",
+    ],
+)
+
 cc_test(
     name = "tensorflow_session_from_frozen_graph_generator_test",
     srcs = ["tensorflow_session_from_frozen_graph_generator_test.cc"],
@@ -901,6 +1038,7 @@ cc_test(
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:rectangle",
+        "//mediapipe/util:audio_decoder_cc_proto",
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
@@ -908,6 +1046,20 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "vector_int_to_tensor_calculator_test",
+    srcs = ["vector_int_to_tensor_calculator_test.cc"],
+    deps = [
+        ":vector_int_to_tensor_calculator",
+        ":vector_int_to_tensor_calculator_options_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+)
+
 cc_test(
     name = "vector_float_to_tensor_calculator_test",
     srcs = ["vector_float_to_tensor_calculator_test.cc"],

mediapipe/calculators/tensorflow/graph_tensors_packet_generator_test.cc

@@ -74,7 +74,7 @@ TEST_F(GraphTensorsPacketGeneratorTest, VerifyTensorSizeShapeAndValue) {
 
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "GraphTensorsPacketGenerator", extendable_options_, inputs, &outputs);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   VerifyTensorMap(&outputs);
 }
 

mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator_test.cc

@@ -171,7 +171,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, SolidRedRGBFrame) {
     runner_ = ::absl::make_unique<CalculatorRunner>(
         "ImageFrameToTensorCalculator", "", 1, 1, 0);
     AddRGBFrame(width, height);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -212,7 +212,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, SolidRedRGBAFrame) {
     runner_.reset(
         new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
     AddRGBAFrame(width, height);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -254,7 +254,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, SolidGray8Frame) {
     runner_.reset(
         new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
     AddGray8Frame(width, height);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -293,7 +293,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, SolidGray16Frame) {
     runner_.reset(
         new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
     AddGray16Frame(width, height);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -332,7 +332,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, SolidFloatFrame) {
     runner_.reset(
         new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
     AddFloatFrame(width, height);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -363,7 +363,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, FixedNoiseRGBFrame) {
   runner_.reset(
       new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
   AddFixedNoiseRGBFrame();
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Index(0).packets;
   ASSERT_EQ(1, output_packets.size());
@@ -396,7 +396,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, RandomRGBFrame) {
     runner_.reset(
         new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0));
     AddRandomRGBFrame(width, height, seed);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -440,7 +440,7 @@ TEST_F(ImageFrameToTensorCalculatorTest, FixedRGBFrameWithMeanAndStddev) {
 
   runner_->MutableInputs()->Index(0).packets.push_back(
       Adopt(image_frame.release()).At(Timestamp(0)));
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const auto& tensor = runner_->Outputs().Index(0).packets[0].Get<tf::Tensor>();
   EXPECT_EQ(tensor.dtype(), tf::DT_FLOAT);

mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc

@@ -29,6 +29,11 @@
 
 namespace mediapipe {
 
+const char kBufferSize[] = "BUFFER_SIZE";
+const char kOverlap[] = "OVERLAP";
+const char kTimestampOffset[] = "TIMESTAMP_OFFSET";
+const char kCalculatorOptions[] = "CALCULATOR_OPTIONS";
+
 namespace tf = tensorflow;
 
 // Given an input stream of tensors, concatenates the tensors over timesteps.
@@ -72,6 +77,9 @@ class LappedTensorBufferCalculator : public CalculatorBase {
   ::mediapipe::Status AddBatchDimension(tf::Tensor* input_tensor);
 
   int steps_until_output_;
+  int buffer_size_;
+  int overlap_;
+  int timestamp_offset_;
   std::unique_ptr<CircularBuffer<Timestamp>> timestamp_buffer_;
   std::unique_ptr<CircularBuffer<tf::Tensor>> buffer_;
   LappedTensorBufferCalculatorOptions options_;
@@ -87,6 +95,21 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
   );
   RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
       << "Only one output stream is supported.";
+
+  if (cc->InputSidePackets().HasTag(kBufferSize)) {
+    cc->InputSidePackets().Tag(kBufferSize).Set<int>();
+  }
+  if (cc->InputSidePackets().HasTag(kOverlap)) {
+    cc->InputSidePackets().Tag(kOverlap).Set<int>();
+  }
+  if (cc->InputSidePackets().HasTag(kTimestampOffset)) {
+    cc->InputSidePackets().Tag(kTimestampOffset).Set<int>();
+  }
+  if (cc->InputSidePackets().HasTag(kCalculatorOptions)) {
+    cc->InputSidePackets()
+        .Tag(kCalculatorOptions)
+        .Set<LappedTensorBufferCalculatorOptions>();
+  }
   cc->Outputs().Index(0).Set<tf::Tensor>(
       // Output tensorflow::Tensor stream with possibly overlapping steps.
   );
@@ -95,16 +118,33 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
 
 ::mediapipe::Status LappedTensorBufferCalculator::Open(CalculatorContext* cc) {
   options_ = cc->Options<LappedTensorBufferCalculatorOptions>();
-  RET_CHECK_LT(options_.overlap(), options_.buffer_size());
+  if (cc->InputSidePackets().HasTag(kCalculatorOptions)) {
-  RET_CHECK_GE(options_.timestamp_offset(), 0)
+    options_ = cc->InputSidePackets()
+                   .Tag(kCalculatorOptions)
+                   .Get<LappedTensorBufferCalculatorOptions>();
+  }
+  buffer_size_ = options_.buffer_size();
+  if (cc->InputSidePackets().HasTag(kBufferSize)) {
+    buffer_size_ = cc->InputSidePackets().Tag(kBufferSize).Get<int>();
+  }
+  overlap_ = options_.overlap();
+  if (cc->InputSidePackets().HasTag(kOverlap)) {
+    overlap_ = cc->InputSidePackets().Tag(kOverlap).Get<int>();
+  }
+  timestamp_offset_ = options_.timestamp_offset();
+  if (cc->InputSidePackets().HasTag(kTimestampOffset)) {
+    timestamp_offset_ = cc->InputSidePackets().Tag(kTimestampOffset).Get<int>();
+  }
+
+  RET_CHECK_LT(overlap_, buffer_size_);
+  RET_CHECK_GE(timestamp_offset_, 0)
       << "Negative timestamp_offset is not allowed.";
-  RET_CHECK_LT(options_.timestamp_offset(), options_.buffer_size())
+  RET_CHECK_LT(timestamp_offset_, buffer_size_)
       << "output_frame_num_offset has to be less than buffer_size.";
   timestamp_buffer_ =
-      absl::make_unique<CircularBuffer<Timestamp>>(options_.buffer_size());
+      absl::make_unique<CircularBuffer<Timestamp>>(buffer_size_);
-  buffer_ =
+  buffer_ = absl::make_unique<CircularBuffer<tf::Tensor>>(buffer_size_);
-      absl::make_unique<CircularBuffer<tf::Tensor>>(options_.buffer_size());
+  steps_until_output_ = buffer_size_;
-  steps_until_output_ = options_.buffer_size();
   return ::mediapipe::OkStatus();
 }
 
@@ -128,11 +168,10 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
         concatenated.get());
     RET_CHECK(concat_status.ok()) << concat_status.ToString();
 
-    cc->Outputs().Index(0).Add(
+    cc->Outputs().Index(0).Add(concatenated.release(),
-        concatenated.release(),
+                               timestamp_buffer_->Get(timestamp_offset_));
-        timestamp_buffer_->Get(options_.timestamp_offset()));
 
-    steps_until_output_ = options_.buffer_size() - options_.overlap();
+    steps_until_output_ = buffer_size_ - overlap_;
   }
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc

@@ -74,7 +74,7 @@ TEST_F(MatrixToTensorCalculatorTest, RandomMatrix) {
     runner_ = ::absl::make_unique<CalculatorRunner>("MatrixToTensorCalculator",
                                                     "", 1, 1, 0);
     AddRandomMatrix(num_rows, num_columns, kSeed);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -106,7 +106,7 @@ TEST_F(MatrixToTensorCalculatorTest, RandomMatrixTranspose) {
     runner_ = ::absl::make_unique<CalculatorRunner>(
         "MatrixToTensorCalculator", kTransposeOptionsString, 1, 1, 0);
     AddRandomMatrix(num_rows, num_columns, kSeed);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());
@@ -138,7 +138,7 @@ TEST_F(MatrixToTensorCalculatorTest, RandomMatrixAddDimension) {
     runner_ = ::absl::make_unique<CalculatorRunner>(
         "MatrixToTensorCalculator", kAddDimensionOptionsString, 1, 1, 0);
     AddRandomMatrix(num_rows, num_columns, kSeed);
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     const std::vector<Packet>& output_packets =
         runner_->Outputs().Index(0).packets;
     ASSERT_EQ(1, output_packets.size());

mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc

@@ -134,7 +134,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
     runner_->MutableInputs()->Tag(kClasses).packets.push_back(
         PointToForeign(&input_classes_).At(Timestamp::PostStream()));
 
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size());
   }
 
@@ -146,7 +146,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
         PointToForeign(&input_scores_for_all_classes_)
             .At(Timestamp::PostStream()));
 
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size());
   }
 
@@ -167,7 +167,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
         .packets.push_back(
             PointToForeign(&input_keypoints_).At(Timestamp::PostStream()));
 
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size());
   }
 
@@ -201,7 +201,7 @@ class ObjectDetectionTensorsToDetectionsCalculatorTest
     runner_->MutableInputs()->Tag(kClasses).packets.push_back(
         PointToForeign(&input_classes_).At(Timestamp::PostStream()));
 
-    MEDIAPIPE_ASSERT_OK(runner_->Run());
+    MP_ASSERT_OK(runner_->Run());
     ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size());
   }
 

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -285,6 +285,10 @@ class PackMediaSequenceCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
+    int image_height = -1;
+    int image_width = -1;
+    // Because the tag order may vary, we need to loop through tags to get
+    // image information before processing other tag types.
     for (const auto& tag : cc->Inputs().GetTags()) {
       if (!cc->Inputs().Tag(tag).IsEmpty()) {
         features_present_[tag] = true;
@@ -306,14 +310,21 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
                  << "No encoded image";
         }
+        image_height = image.height();
+        image_width = image.width();
         mpms::AddImageTimestamp(key, cc->InputTimestamp().Value(),
                                 sequence_.get());
         mpms::AddImageEncoded(key, image.encoded_image(), sequence_.get());
       }
+    }
+    for (const auto& tag : cc->Inputs().GetTags()) {
+      if (!cc->Inputs().Tag(tag).IsEmpty()) {
+        features_present_[tag] = true;
+      }
       if (absl::StartsWith(tag, kKeypointsTag) &&
           !cc->Inputs().Tag(tag).IsEmpty()) {
         std::string key = "";
-        if (tag != kImageTag) {
+        if (tag != kKeypointsTag) {
           int tag_length = sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1;
           if (tag[tag_length] == '_') {
             key = tag.substr(tag_length + 1);
@@ -363,11 +374,20 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                   LocationData::BOUNDING_BOX ||
               detection.location_data().format() ==
                   LocationData::RELATIVE_BOUNDING_BOX) {
-            int height = mpms::GetImageHeight(*sequence_);
+            if (mpms::HasImageHeight(*sequence_) &&
-            int width = mpms::GetImageWidth(*sequence_);
+                mpms::HasImageWidth(*sequence_)) {
+              image_height = mpms::GetImageHeight(*sequence_);
+              image_width = mpms::GetImageWidth(*sequence_);
+            }
+            if (image_height == -1 || image_width == -1) {
+              return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+                     << "Images must be provided with bounding boxes or the "
+                        "image "
+                     << "height and width must already be in the example.";
+            }
             Location relative_bbox = Location::CreateRelativeBBoxLocation(
                 Location(detection.location_data())
-                    .ConvertToRelativeBBox(width, height));
+                    .ConvertToRelativeBBox(image_width, image_height));
             predicted_locations.push_back(relative_bbox);
             if (detection.label_size() > 0) {
               predicted_class_strings.push_back(detection.label(0));

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -87,7 +87,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -131,7 +131,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -169,7 +169,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -214,7 +214,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
   runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
       Adopt(image_ptr.release()).At(Timestamp(0)));
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -257,7 +257,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -321,7 +321,149 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence));
+  ASSERT_EQ(height, mpms::GetImageHeight(output_sequence));
+  ASSERT_EQ(width, mpms::GetImageWidth(output_sequence));
+  ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxSize(output_sequence));
+  ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxTimestampSize(output_sequence));
+  ASSERT_EQ(0, mpms::GetClassSegmentationEncodedSize(output_sequence));
+  ASSERT_EQ(0, mpms::GetClassSegmentationTimestampSize(output_sequence));
+  for (int i = 0; i < num_vectors; ++i) {
+    ASSERT_EQ(i, mpms::GetPredictedBBoxTimestampAt(output_sequence, i));
+    auto bboxes = mpms::GetPredictedBBoxAt(output_sequence, i);
+    ASSERT_EQ(2, bboxes.size());
+    for (int j = 0; j < bboxes.size(); ++j) {
+      auto rect = bboxes[j].GetRelativeBBox();
+      ASSERT_NEAR(0, rect.xmin(), 0.001);
+      ASSERT_NEAR(0.5, rect.ymin(), 0.001);
+      ASSERT_NEAR(0.5, rect.xmax(), 0.001);
+      ASSERT_NEAR(1.0, rect.ymax(), 0.001);
+    }
+    auto class_strings =
+        mpms::GetPredictedBBoxLabelStringAt(output_sequence, i);
+    ASSERT_EQ("absolute bbox", class_strings[0]);
+    ASSERT_EQ("relative bbox", class_strings[1]);
+    auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i);
+    ASSERT_EQ(0, class_indices[0]);
+    ASSERT_EQ(1, class_indices[1]);
+  }
+}
+
+TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) {
+  SetUpCalculator({"BBOX_PREDICTED:detections"}, {}, false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+  std::string test_video_id = "test_video_id";
+  mpms::SetClipMediaId(test_video_id, input_sequence.get());
+  int height = 480;
+  int width = 640;
+  int num_vectors = 2;
+  for (int i = 0; i < num_vectors; ++i) {
+    auto detections = ::absl::make_unique<::std::vector<Detection>>();
+    Detection detection;
+    detection.add_label("absolute bbox");
+    detection.add_label_id(0);
+    detection.add_score(0.5);
+    Location::CreateBBoxLocation(0, height / 2, width / 2, height / 2)
+        .ConvertToProto(detection.mutable_location_data());
+    detections->push_back(detection);
+
+    detection = Detection();
+    detection.add_label("relative bbox");
+    detection.add_label_id(1);
+    detection.add_score(0.75);
+    Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
+        .ConvertToProto(detection.mutable_location_data());
+    detections->push_back(detection);
+
+    // The mask detection should be ignored in the output.
+    detection = Detection();
+    detection.add_label("mask");
+    detection.add_score(1.0);
+    cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
+    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+        detection.mutable_location_data());
+    detections->push_back(detection);
+
+    runner_->MutableInputs()
+        ->Tag("BBOX_PREDICTED")
+        .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
+  }
+
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+
+  auto status = runner_->Run();
+  EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
+}
+
+TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
+  SetUpCalculator({"BBOX_PREDICTED:detections", "IMAGE:images"}, {}, false,
+                  true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+  std::string test_video_id = "test_video_id";
+  mpms::SetClipMediaId(test_video_id, input_sequence.get());
+  int height = 480;
+  int width = 640;
+  int num_vectors = 2;
+  for (int i = 0; i < num_vectors; ++i) {
+    auto detections = ::absl::make_unique<::std::vector<Detection>>();
+    Detection detection;
+    detection.add_label("absolute bbox");
+    detection.add_label_id(0);
+    detection.add_score(0.5);
+    Location::CreateBBoxLocation(0, height / 2, width / 2, height / 2)
+        .ConvertToProto(detection.mutable_location_data());
+    detections->push_back(detection);
+
+    detection = Detection();
+    detection.add_label("relative bbox");
+    detection.add_label_id(1);
+    detection.add_score(0.75);
+    Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5)
+        .ConvertToProto(detection.mutable_location_data());
+    detections->push_back(detection);
+
+    // The mask detection should be ignored in the output.
+    detection = Detection();
+    detection.add_label("mask");
+    detection.add_score(1.0);
+    cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
+    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+        detection.mutable_location_data());
+    detections->push_back(detection);
+
+    runner_->MutableInputs()
+        ->Tag("BBOX_PREDICTED")
+        .packets.push_back(Adopt(detections.release()).At(Timestamp(i)));
+  }
+  cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
+  std::vector<uchar> bytes;
+  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  std::string test_image_string(bytes.begin(), bytes.end());
+  OpenCvImageEncoderCalculatorResults encoded_image;
+  encoded_image.set_encoded_image(test_image_string);
+  encoded_image.set_width(width);
+  encoded_image.set_height(height);
+
+  int num_images = 2;
+  for (int i = 0; i < num_images; ++i) {
+    auto image_ptr =
+        ::absl::make_unique<OpenCvImageEncoderCalculatorResults>(encoded_image);
+    runner_->MutableInputs()->Tag("IMAGE").packets.push_back(
+        Adopt(image_ptr.release()).At(Timestamp(i)));
+  }
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -374,7 +516,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoKeypoints) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -424,7 +566,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -473,7 +615,7 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -536,7 +678,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -562,7 +704,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFlowImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -599,7 +741,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFloatVectors) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
@@ -643,7 +785,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets;
   ASSERT_EQ(1, output_packets.size());

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -34,6 +34,10 @@
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_util.h"
 
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+#include "tensorflow/core/profiler/lib/traceme.h"
+#endif
+
 namespace tf = ::tensorflow;
 
 namespace mediapipe {
@@ -361,14 +365,14 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
     }
 
     if (batch_timestamps_.size() == options_.batch_size()) {
-      RETURN_IF_ERROR(OutputBatch(cc));
+      MP_RETURN_IF_ERROR(OutputBatch(cc));
     }
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Close(CalculatorContext* cc) override {
     if (!batch_timestamps_.empty()) {
-      RETURN_IF_ERROR(OutputBatch(cc));
+      MP_RETURN_IF_ERROR(OutputBatch(cc));
     }
     return ::mediapipe::OkStatus();
   }
@@ -435,9 +439,15 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
       session_run_throttle->Acquire(1);
     }
     const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
-    const tf::Status tf_status =
+    tf::Status tf_status;
-        session_->Run(input_tensors, output_tensor_names,
+    {
-                      {} /* target_node_names */, &outputs);
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+      tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
+#endif
+      tf_status = session_->Run(input_tensors, output_tensor_names,
+                                {} /* target_node_names */, &outputs);
+    }
+
     if (session_run_throttle != nullptr) {
       session_run_throttle->Release(1);
     }

mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc

@@ -122,7 +122,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetConstants) {
   runner_ = absl::make_unique<CalculatorRunner>(config);
   AddSessionInputSidePacket();
   AddVectorToInputsAsTensor({0, 0, 0}, "A", 0);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_b =
       runner_->Outputs().Tag("B").packets;
@@ -163,7 +163,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetComputed) {
   AddSessionInputSidePacket();
   AddVectorToInputsAsTensor({2, 2, 2}, "A", 0);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -217,7 +217,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed) {
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
   AddVectorToInputsAsTensor({3, 3, 3}, "A", 1);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -255,7 +255,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetSingleBatchComputed) {
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
   AddVectorToInputsAsTensor({3, 3, 3}, "A", 1);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -293,7 +293,7 @@ TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputed) {
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
   AddVectorToInputsAsTensor({3, 3, 3}, "A", 1);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -331,7 +331,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) {
   AddSessionInputSidePacket();
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -372,7 +372,7 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) {
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
   AddVectorToInputsAsTensor({1, 1, 1}, "A", 1);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -409,7 +409,7 @@ TEST_F(TensorflowInferenceCalculatorTest, DISABLED_CheckTiming) {
   runner_ = absl::make_unique<CalculatorRunner>(config);
   AddSessionInputSidePacket();
   AddVectorToInputsAsTensor({0, 0, 0}, "A", 0);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   EXPECT_EQ(1, runner_
                    ->GetCounter(
@@ -465,7 +465,7 @@ TEST_F(TensorflowInferenceCalculatorTest, MissingInputFeature_Skip) {
   runner_ = absl::make_unique<CalculatorRunner>(config);
   AddSessionInputSidePacket();
   AddVectorToInputsAsTensor({2, 2, 2}, "A", 0);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;
@@ -494,7 +494,7 @@ TEST_F(TensorflowInferenceCalculatorTest,
   AddVectorToInputsAsTensor({2, 2, 2}, "A", 0);
   AddVectorToInputsAsTensor({3, 3, 3}, "A", 1);
   AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets_mult =
       runner_->Outputs().Tag("MULTIPLIED").packets;

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -0,0 +1,136 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Reads serialized GraphDef proto. There are three ways to load a model:
+// 1. Specify the path to a graph.pb in the calculator options.
+// 2. Specify the path to the graph.pb through the
+// input_side_packet:STRING_MODEL_FILE_PATH
+// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL,
+// typically provided by EmbeddingFilePacketFactory.
+//
+// Produces a SessionBundle that TensorFlowInferenceCalculator can use.
+
+#include <string>
+
+#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "tensorflow/core/public/session_options.h"
+
+#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
+    defined(__APPLE__) && !TARGET_OS_OSX
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+namespace tf = ::tensorflow;
+
+class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    const auto& options =
+        cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
+    bool has_exactly_one_model =
+        !options.graph_proto_path().empty()
+            ? !(cc->InputSidePackets().HasTag("STRING_MODEL") |
+                cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"))
+            : (cc->InputSidePackets().HasTag("STRING_MODEL") ^
+               cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"));
+    RET_CHECK(has_exactly_one_model)
+        << "Must have exactly one of graph_proto_path in options or "
+           "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
+    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+      cc->InputSidePackets()
+          .Tag("STRING_MODEL")
+          .Set<std::string>(
+              // String model from embedded path
+          );
+    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+      cc->InputSidePackets()
+          .Tag("STRING_MODEL_FILE_PATH")
+          .Set<std::string>(
+              // Filename of std::string model.
+          );
+    }
+    cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>(
+        // A TensorFlow model loaded and ready for use along with
+        // a map from tags to tensor names.
+    );
+    RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const auto& options =
+        cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
+    // Output bundle packet.
+    auto session = ::absl::make_unique<TensorFlowSession>();
+
+    tf::SessionOptions session_options;
+    session_options.config.CopyFrom(options.config());
+    std::vector<mediapipe::ProtoString> initialization_op_names;
+    initialization_op_names.reserve(options.initialization_op_names_size());
+    for (int i = 0; i < options.initialization_op_names_size(); ++i) {
+      initialization_op_names.emplace_back(options.initialization_op_names(i));
+    }
+    session->session.reset(tf::NewSession(session_options));
+
+    std::string graph_def_serialized;
+    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+      graph_def_serialized =
+          cc->InputSidePackets().Tag("STRING_MODEL").Get<std::string>();
+    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+      const std::string& frozen_graph = cc->InputSidePackets()
+                                            .Tag("STRING_MODEL_FILE_PATH")
+                                            .Get<std::string>();
+      RET_CHECK_OK(
+          mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
+    } else {
+      RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(),
+                                                &graph_def_serialized));
+    }
+    tensorflow::GraphDef graph_def;
+
+    RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
+    const tf::Status tf_status = session->session->Create(graph_def);
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+
+    for (const auto& key_value : options.tag_to_tensor_names()) {
+      session->tag_to_tensor_map[key_value.first] = key_value.second;
+    }
+    if (!initialization_op_names.empty()) {
+      const tf::Status tf_status =
+          session->session->Run({}, {}, initialization_op_names, {});
+      // RET_CHECK on the tf::Status object itself in order to print an
+      // informative error message.
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+    }
+
+    cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+};
+REGISTER_CALCULATOR(TensorFlowSessionFromFrozenGraphCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto

@@ -0,0 +1,72 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "tensorflow/core/protobuf/config.proto";
+
+message TensorFlowSessionFromFrozenGraphCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional TensorFlowSessionFromFrozenGraphCalculatorOptions ext = 266997877;
+  }
+
+  // Path to file containing serialized proto of type tensorflow::GraphDef.
+  optional string graph_proto_path = 1;
+
+  // To run inference with MediaPipe inputs MediaPipe streams need to be mapped
+  // to TensorFlow tensors. This map defines the which streams are fed into
+  // which tensors in the model. The MediaPipe tag of the stream is the map key.
+  // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING"
+  // and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of
+  // input_stream or output_stream of the TensorflowInferenceCalculator
+  // consuming the packet produced by this calculator. The tensor names must
+  // match the tensor names in the graph that you want to feed or fetch into or
+  // out of. Examples:  "DecodeJpeg/contents:0" or "softmax:0". For example, a
+  // mediapipe graph can include the nodes:
+  //
+  // node {
+  //   calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+  //   output_side_packet: "SESSION:session"
+  //   options {
+  //     [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+  //       graph_proto_path: "[PATH]"
+  //       tag_to_tensor_names {
+  //         key: "JPG_STRING"
+  //         value: "input:0"
+  //       }
+  //       tag_to_tensor_names {
+  //         key: "SOFTMAX"
+  //         value: "softmax:0"
+  //       }
+  //     }
+  //   }
+  // }
+  // node {
+  //   calculator: "TensorflowInferenceCalculator"
+  //   input_side_packet: "SESSION:graph_with_bindings"
+  //   input_stream: "JPG_STRING:jpg_string_tensor"
+  //   output_stream: "SOFTMAX:softmax_tensor"
+  // }
+  map<string, string> tag_to_tensor_names = 2;
+
+  // Tensorflow session config options.
+  optional tensorflow.ConfigProto config = 3;
+
+  // Graph nodes to run to initialize the model. Any output of these ops is
+  // ignored.
+  repeated string initialization_op_names = 4;
+}

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc

@@ -0,0 +1,316 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/tag_map_helper.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+namespace tf = ::tensorflow;
+
+std::string GetGraphDefPath() {
+  return mediapipe::file::JoinPath("./",
+                                   "mediapipe/calculators/tensorflow/"
+                                   "testdata/frozen_graph_def.pb");
+}
+
+// Helper function that creates Tensor INT32 matrix with size 1x3.
+tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) {
+  tf::Tensor tensor(tf::DT_INT32,
+                    tf::TensorShape(std::vector<tf::int64>({1, 3})));
+  auto matrix = tensor.matrix<int32>();
+  matrix(0, 0) = v1;
+  matrix(0, 1) = v2;
+  matrix(0, 2) = v3;
+  return tensor;
+}
+
+class TensorFlowSessionFromFrozenGraphCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    extendable_options_.Clear();
+    calculator_options_ = extendable_options_.MutableExtension(
+        TensorFlowSessionFromFrozenGraphCalculatorOptions::ext);
+    calculator_options_->set_graph_proto_path(GetGraphDefPath());
+    (*calculator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] =
+        "multiplied:0";
+    (*calculator_options_->mutable_tag_to_tensor_names())["A"] = "a:0";
+    (*calculator_options_->mutable_tag_to_tensor_names())["B"] = "b:0";
+    calculator_options_->mutable_config()->set_intra_op_parallelism_threads(1);
+    calculator_options_->mutable_config()->set_inter_op_parallelism_threads(2);
+  }
+
+  void VerifySignatureMap(const TensorFlowSession& session) {
+    // Session must be set.
+    ASSERT_NE(session.session, nullptr);
+
+    // Bindings are inserted.
+    EXPECT_EQ(session.tag_to_tensor_map.size(), 3);
+
+    // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators.
+    EXPECT_FALSE(session.tag_to_tensor_map.find("A") ==
+                 session.tag_to_tensor_map.end());
+    EXPECT_FALSE(session.tag_to_tensor_map.find("B") ==
+                 session.tag_to_tensor_map.end());
+    EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") ==
+                 session.tag_to_tensor_map.end());
+    // Sanity: find() actually returns a reference to end() if element not
+    // found.
+    EXPECT_TRUE(session.tag_to_tensor_map.find("Z") ==
+                session.tag_to_tensor_map.end());
+
+    EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0");
+    EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0");
+    EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0");
+  }
+
+  CalculatorOptions extendable_options_;
+  TensorFlowSessionFromFrozenGraphCalculatorOptions* calculator_options_;
+};
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CreatesPacketWithGraphAndBindings) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:tf_model"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+
+  MP_ASSERT_OK(runner.Run());
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+// Integration test. Verifies that TensorFlowInferenceCalculator correctly
+// consumes the Packet emitted by this calculator.
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       ProducesPacketUsableByTensorFlowInferenceCalculator) {
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          absl::Substitute(R"(
+      node {
+        calculator: "TensorFlowInferenceCalculator"
+        input_side_packet: "SESSION:session"
+        input_stream: "A:a_tensor"
+        output_stream: "MULTIPLIED:multiplied_tensor"
+        options {
+          [mediapipe.TensorFlowInferenceCalculatorOptions.ext] {
+            batch_size: 5
+            add_batch_dim_to_tensors: false
+          }
+        }
+      }
+
+      node {
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        }
+      }
+      input_stream: "a_tensor"
+  )",
+                           calculator_options_->DebugString()));
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  StatusOrPoller status_or_poller =
+      graph.AddOutputStreamPoller("multiplied_tensor");
+  ASSERT_TRUE(status_or_poller.ok());
+  OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
+
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "a_tensor",
+      Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.CloseInputStream("a_tensor"));
+
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  // input tensor gets multiplied by [[3, 2, 1]]. Expected output:
+  tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10);
+  EXPECT_EQ(expected_multiplication.DebugString(),
+            packet.Get<tf::Tensor>().DebugString());
+
+  ASSERT_FALSE(poller.Next(&packet));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CreatesPacketWithGraphAndBindingsFromInputSidePacket) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+
+  std::string serialized_graph_contents;
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                            &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  MP_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+TEST_F(
+    TensorFlowSessionFromFrozenGraphCalculatorTest,
+    CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  MP_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForOptionsAndInputsProvideGraphDefProto) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForAllInputsProvideGraphDefProto) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  std::string serialized_graph_contents;
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                            &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  std::string serialized_graph_contents;
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                            &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckInitializationOpName) {
+  calculator_options_->add_initialization_op_names("multiplied:0");
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  MP_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc

@@ -106,7 +106,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   VerifySignatureMap(&output_side_packets);
 }
 
@@ -144,17 +144,17 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
                            generator_options_->DebugString()));
 
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.Initialize(config));
   StatusOrPoller status_or_poller =
       graph.AddOutputStreamPoller("multiplied_tensor");
   ASSERT_TRUE(status_or_poller.ok());
   OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
 
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "a_tensor",
       Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor"));
+  MP_ASSERT_OK(graph.CloseInputStream("a_tensor"));
 
   Packet packet;
   ASSERT_TRUE(poller.Next(&packet));
@@ -164,7 +164,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
             packet.Get<tf::Tensor>().DebugString());
 
   ASSERT_FALSE(poller.Next(&packet));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
@@ -174,15 +174,15 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).ValueOrDie());
   std::string serialized_graph_contents;
-  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
-                                                   &serialized_graph_contents));
+                                            &serialized_graph_contents));
   generator_options_->clear_graph_proto_path();
   input_side_packets.Tag("STRING_MODEL") =
       Adopt(new std::string(serialized_graph_contents));
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   VerifySignatureMap(&output_side_packets);
 }
 
@@ -199,7 +199,7 @@ TEST_F(
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   VerifySignatureMap(&output_side_packets);
 }
 
@@ -229,8 +229,8 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).ValueOrDie());
   std::string serialized_graph_contents;
-  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
-                                                   &serialized_graph_contents));
+                                            &serialized_graph_contents));
   input_side_packets.Tag("STRING_MODEL") =
       Adopt(new std::string(serialized_graph_contents));
   input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
@@ -254,8 +254,8 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   PacketSet output_side_packets(
       tool::CreateTagMap({"SESSION:session"}).ValueOrDie());
   std::string serialized_graph_contents;
-  EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+  MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
-                                         &serialized_graph_contents));
+                                            &serialized_graph_contents));
   input_side_packets.Tag("STRING_MODEL") =
       Adopt(new std::string(serialized_graph_contents));
   input_side_packets.Tag("STRING_MODEL_FILE_PATH") =
@@ -280,7 +280,7 @@ TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest,
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status);
+  MP_EXPECT_OK(run_status);
   VerifySignatureMap(&output_side_packets);
 }
 

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -14,10 +14,6 @@
 
 #include <algorithm>
 
-#if defined(MEDIAPIPE_TPU_SUPPORT)
-#include "learning/brain/google/xla/global_tpu_init.h"
-#include "tensorflow/core/protobuf/tpu/topology.pb.h"
-#endif
 #if !defined(__ANDROID__)
 #include "mediapipe/framework/port/file_helpers.h"
 #endif

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc

@@ -75,7 +75,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
           }
         })",
                                            options_->DebugString()));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
       runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.
@@ -119,7 +119,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
                                            options_->DebugString()));
   runner.MutableSidePackets()->Tag("STRING_SAVED_MODEL_PATH") =
       MakePacket<std::string>(GetSavedModelDir());
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
       runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.
@@ -159,17 +159,17 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
                            options_->DebugString()));
 
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.Initialize(graph_config));
   StatusOrPoller status_or_poller =
       graph.AddOutputStreamPoller("multiplied_tensor");
   ASSERT_TRUE(status_or_poller.ok());
   OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
 
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "a_tensor",
       Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor"));
+  MP_ASSERT_OK(graph.CloseInputStream("a_tensor"));
 
   Packet packet;
   ASSERT_TRUE(poller.Next(&packet));
@@ -179,7 +179,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
             packet.Get<tf::Tensor>().DebugString());
 
   ASSERT_FALSE(poller.Next(&packet));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
@@ -197,7 +197,7 @@ TEST_F(TensorFlowSessionFromSavedModelCalculatorTest,
           }
         })",
                                            options_->DebugString()));
-  MEDIAPIPE_ASSERT_OK(runner.Run());
+  MP_ASSERT_OK(runner.Run());
   const TensorFlowSession& session =
       runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -14,10 +14,6 @@
 
 #include <algorithm>
 
-#if defined(MEDIAPIPE_TPU_SUPPORT)
-#include "learning/brain/google/xla/global_tpu_init.h"
-#include "tensorflow/core/protobuf/tpu/topology.pb.h"
-#endif
 #if !defined(__ANDROID__)
 #include "mediapipe/framework/port/file_helpers.h"
 #endif

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc

@@ -71,7 +71,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromSavedModelGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
       output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.
@@ -113,7 +113,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromSavedModelGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
       output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.
@@ -154,17 +154,17 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
                            generator_options_->DebugString()));
 
   CalculatorGraph graph;
-  MEDIAPIPE_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.Initialize(graph_config));
   StatusOrPoller status_or_poller =
       graph.AddOutputStreamPoller("multiplied_tensor");
   ASSERT_TRUE(status_or_poller.ok());
   OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
 
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "a_tensor",
       Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor"));
+  MP_ASSERT_OK(graph.CloseInputStream("a_tensor"));
 
   Packet packet;
   ASSERT_TRUE(poller.Next(&packet));
@@ -174,7 +174,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
             packet.Get<tf::Tensor>().DebugString());
 
   ASSERT_FALSE(poller.Next(&packet));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
@@ -189,7 +189,7 @@ TEST_F(TensorFlowSessionFromSavedModelGeneratorTest,
   ::mediapipe::Status run_status = tool::RunGenerateAndValidateTypes(
       "TensorFlowSessionFromSavedModelGenerator", extendable_options_,
       input_side_packets, &output_side_packets);
-  MEDIAPIPE_EXPECT_OK(run_status) << run_status.message();
+  MP_EXPECT_OK(run_status) << run_status.message();
   const TensorFlowSession& session =
       output_side_packets.Tag("SESSION").Get<TensorFlowSession>();
   // Session must be set.

mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc

@@ -0,0 +1,126 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "tensorflow/core/example/example.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/io/record_reader.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system.h"
+
+namespace mediapipe {
+
+const char kTFRecordPath[] = "TFRECORD_PATH";
+const char kRecordIndex[] = "RECORD_INDEX";
+const char kExampleTag[] = "EXAMPLE";
+const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
+
+// Reads a tensorflow example/sequence example from a tfrecord file.
+// If the "RECORD_INDEX" input side packet is provided, the calculator is going
+// to fetch the example/sequence example of the tfrecord file at the target
+// record index. Otherwise, the reader always reads the first example/sequence
+// example of the tfrecord file.
+//
+// Example config:
+// node {
+//   calculator: "TFRecordReaderCalculator"
+//   input_side_packet: "TFRECORD_PATH:tfrecord_path"
+//   input_side_packet: "RECORD_INDEX:record_index"
+//   output_side_packet: "SEQUENCE_EXAMPLE:sequence_example"
+// }
+class TFRecordReaderCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+
+::mediapipe::Status TFRecordReaderCalculator::GetContract(
+    CalculatorContract* cc) {
+  cc->InputSidePackets().Tag(kTFRecordPath).Set<std::string>();
+  if (cc->InputSidePackets().HasTag(kRecordIndex)) {
+    cc->InputSidePackets().Tag(kRecordIndex).Set<int>();
+  }
+
+  RET_CHECK(cc->OutputSidePackets().HasTag(kExampleTag) ||
+            cc->OutputSidePackets().HasTag(kSequenceExampleTag))
+      << "TFRecordReaderCalculator must output either Tensorflow example or "
+         "sequence example.";
+  if (cc->OutputSidePackets().HasTag(kExampleTag)) {
+    cc->OutputSidePackets().Tag(kExampleTag).Set<tensorflow::Example>();
+  } else {
+    cc->OutputSidePackets()
+        .Tag(kSequenceExampleTag)
+        .Set<tensorflow::SequenceExample>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TFRecordReaderCalculator::Open(CalculatorContext* cc) {
+  std::unique_ptr<tensorflow::RandomAccessFile> file;
+  auto tf_status = tensorflow::Env::Default()->NewRandomAccessFile(
+      cc->InputSidePackets().Tag(kTFRecordPath).Get<std::string>(), &file);
+  RET_CHECK(tf_status.ok())
+      << "Failed to open tfrecord file: " << tf_status.error_message();
+  tensorflow::io::RecordReader reader(file.get(),
+                                      tensorflow::io::RecordReaderOptions());
+  tensorflow::uint64 offset = 0;
+  std::string example_str;
+  const int target_idx =
+      cc->InputSidePackets().HasTag(kRecordIndex)
+          ? cc->InputSidePackets().Tag(kRecordIndex).Get<int>()
+          : 0;
+  int current_idx = 0;
+  while (current_idx <= target_idx) {
+    tf_status = reader.ReadRecord(&offset, &example_str);
+    RET_CHECK(tf_status.ok())
+        << "Failed to read tfrecord: " << tf_status.error_message();
+    if (current_idx == target_idx) {
+      if (cc->OutputSidePackets().HasTag(kExampleTag)) {
+        tensorflow::Example tf_example;
+        tf_example.ParseFromString(example_str);
+        cc->OutputSidePackets()
+            .Tag(kExampleTag)
+            .Set(MakePacket<tensorflow::Example>(std::move(tf_example)));
+      } else {
+        tensorflow::SequenceExample tf_sequence_example;
+        tf_sequence_example.ParseFromString(example_str);
+        cc->OutputSidePackets()
+            .Tag(kSequenceExampleTag)
+            .Set(MakePacket<tensorflow::SequenceExample>(
+                std::move(tf_sequence_example)));
+      }
+    }
+    ++current_idx;
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TFRecordReaderCalculator::Process(CalculatorContext* cc) {
+  return ::mediapipe::OkStatus();
+}
+
+REGISTER_CALCULATOR(TFRecordReaderCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/formats/location.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/audio_decoder.pb.h"
 #include "mediapipe/util/sequence/media_sequence.h"
 #include "tensorflow/core/example/example.pb.h"
 #include "tensorflow/core/example/feature.pb.h"
@@ -37,6 +38,7 @@ const char kDatasetRootDirTag[] = "DATASET_ROOT";
 const char kDataPath[] = "DATA_PATH";
 const char kPacketResamplerOptions[] = "RESAMPLER_OPTIONS";
 const char kImagesFrameRateTag[] = "IMAGE_FRAME_RATE";
+const char kAudioDecoderOptions[] = "AUDIO_DECODER_OPTIONS";
 
 namespace tf = ::tensorflow;
 namespace mpms = ::mediapipe::mediasequence;
@@ -126,6 +128,11 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
     if (cc->OutputSidePackets().HasTag(kDataPath)) {
       cc->OutputSidePackets().Tag(kDataPath).Set<std::string>();
     }
+    if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions)) {
+      cc->OutputSidePackets()
+          .Tag(kAudioDecoderOptions)
+          .Set<AudioDecoderOptions>();
+    }
     if (cc->OutputSidePackets().HasTag(kImagesFrameRateTag)) {
       cc->OutputSidePackets().Tag(kImagesFrameRateTag).Set<double>();
     }
@@ -136,10 +143,11 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
     }
     if ((options.has_padding_before_label() ||
          options.has_padding_after_label()) &&
-        !(cc->OutputSidePackets().HasTag(kPacketResamplerOptions))) {
+        !(cc->OutputSidePackets().HasTag(kAudioDecoderOptions) ||
+          cc->OutputSidePackets().HasTag(kPacketResamplerOptions))) {
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
-             << "If specifying padding, must output "
+             << "If specifying padding, must output " << kPacketResamplerOptions
-             << kPacketResamplerOptions;
+             << "or" << kAudioDecoderOptions;
     }
 
     // Optional streams.
@@ -260,7 +268,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
     // Set the start and end of the clip in the appropriate options protos.
     double start_time = 0;
     double end_time = 0;
-    if (cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) {
+    if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions) ||
+        cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) {
       if (mpms::HasClipStartTimestamp(sequence)) {
         start_time =
             Timestamp(mpms::GetClipStartTimestamp(sequence)).Seconds() -
@@ -271,6 +280,27 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
                    options.padding_after_label();
       }
     }
+    if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions)) {
+      auto audio_decoder_options = absl::make_unique<AudioDecoderOptions>(
+          options.base_audio_decoder_options());
+      if (mpms::HasClipStartTimestamp(sequence)) {
+        if (options.force_decoding_from_start_of_media()) {
+          audio_decoder_options->set_start_time(0);
+        } else {
+          audio_decoder_options->set_start_time(
+              start_time - options.extra_padding_from_media_decoder());
+        }
+      }
+      if (mpms::HasClipEndTimestamp(sequence)) {
+        audio_decoder_options->set_end_time(
+            end_time + options.extra_padding_from_media_decoder());
+      }
+      LOG(INFO) << "Created AudioDecoderOptions:\n"
+                << audio_decoder_options->DebugString();
+      cc->OutputSidePackets()
+          .Tag(kAudioDecoderOptions)
+          .Set(Adopt(audio_decoder_options.release()));
+    }
     if (cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) {
       auto resampler_options = absl::make_unique<CalculatorOptions>();
       *(resampler_options->MutableExtension(

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto

@@ -18,6 +18,7 @@ package mediapipe;
 
 import "mediapipe/calculators/core/packet_resampler_calculator.proto";
 import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/audio_decoder.proto";
 
 message UnpackMediaSequenceCalculatorOptions {
   extend mediapipe.CalculatorOptions {
@@ -49,4 +50,10 @@ message UnpackMediaSequenceCalculatorOptions {
   // parameters for the MediaDecoderCalculator. End time parameters are still
   // respected.
   optional bool force_decoding_from_start_of_media = 7;
+
+  // Stores the audio decoder settings for the graph. (e.g. which audio
+  // stream to pull from the video.) The sequence's metadata overrides
+  // the clip start and end times and outputs these for the
+  // AudioDecoderCalculator to consume.
+  optional AudioDecoderOptions base_audio_decoder_options = 9;
 }

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc

@@ -23,6 +23,7 @@
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/rectangle.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/audio_decoder.pb.h"
 #include "mediapipe/util/sequence/media_sequence.h"
 #include "tensorflow/core/example/example.pb.h"
 
@@ -97,7 +98,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneImage) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("IMAGE").packets;
@@ -126,7 +127,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("IMAGE").packets;
@@ -156,7 +157,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPrefixedImages) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("IMAGE_PREFIX").packets;
@@ -183,7 +184,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneForwardFlowImage) {
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
@@ -212,7 +213,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoForwardFlowImages) {
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets;
@@ -242,7 +243,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksBBoxes) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("BBOX").packets;
@@ -276,7 +277,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPrefixedBBoxes) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("BBOX_PREFIX").packets;
@@ -308,7 +309,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("FLOAT_FEATURE_TEST").packets;
@@ -353,7 +354,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) {
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& output_packets =
       runner_->Outputs().Tag("IMAGE").packets;
@@ -390,7 +391,7 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) {
 
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(input_sequence.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
   const std::vector<Packet>& fdense_avg_packets =
       runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_AVG").packets;
@@ -419,11 +420,11 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromPacket) {
 
   std::string root = "test_root";
   runner_->MutableSidePackets()->Tag("DATASET_ROOT") = PointToForeign(&root);
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
-  MEDIAPIPE_ASSERT_OK(runner_->OutputSidePackets()
+  MP_ASSERT_OK(runner_->OutputSidePackets()
-                          .Tag("DATA_PATH")
+                   .Tag("DATA_PATH")
-                          .ValidateAsType<std::string>());
+                   .ValidateAsType<std::string>());
   ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
             root + "/" + data_path_);
 }
@@ -437,11 +438,11 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromOptions) {
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(sequence_.release());
 
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
-  MEDIAPIPE_ASSERT_OK(runner_->OutputSidePackets()
+  MP_ASSERT_OK(runner_->OutputSidePackets()
-                          .Tag("DATA_PATH")
+                   .Tag("DATA_PATH")
-                          .ValidateAsType<std::string>());
+                   .ValidateAsType<std::string>());
   ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
             root + "/" + data_path_);
 }
@@ -450,15 +451,71 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromExample) {
   SetUpCalculator({}, {"DATA_PATH:data_path"});
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(sequence_.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
-  MEDIAPIPE_ASSERT_OK(runner_->OutputSidePackets()
+  MP_ASSERT_OK(runner_->OutputSidePackets()
-                          .Tag("DATA_PATH")
+                   .Tag("DATA_PATH")
-                          .ValidateAsType<std::string>());
+                   .ValidateAsType<std::string>());
   ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get<std::string>(),
             data_path_);
 }
 
+TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptions) {
+  CalculatorOptions options;
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_padding_before_label(1);
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_padding_after_label(2);
+  SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
+                  &options);
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(sequence_.release());
+  MP_ASSERT_OK(runner_->Run());
+
+  MP_EXPECT_OK(runner_->OutputSidePackets()
+                   .Tag("AUDIO_DECODER_OPTIONS")
+                   .ValidateAsType<AudioDecoderOptions>());
+  EXPECT_NEAR(runner_->OutputSidePackets()
+                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Get<AudioDecoderOptions>()
+                  .start_time(),
+              2.0, 1e-5);
+  EXPECT_NEAR(runner_->OutputSidePackets()
+                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Get<AudioDecoderOptions>()
+                  .end_time(),
+              7.0, 1e-5);
+}
+
+TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptionsOverride) {
+  CalculatorOptions options;
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_padding_before_label(1);
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_padding_after_label(2);
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_force_decoding_from_start_of_media(true);
+  SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {},
+                  &options);
+  runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
+      Adopt(sequence_.release());
+  MP_ASSERT_OK(runner_->Run());
+
+  MP_EXPECT_OK(runner_->OutputSidePackets()
+                   .Tag("AUDIO_DECODER_OPTIONS")
+                   .ValidateAsType<AudioDecoderOptions>());
+  EXPECT_NEAR(runner_->OutputSidePackets()
+                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Get<AudioDecoderOptions>()
+                  .start_time(),
+              0.0, 1e-5);
+  EXPECT_NEAR(runner_->OutputSidePackets()
+                  .Tag("AUDIO_DECODER_OPTIONS")
+                  .Get<AudioDecoderOptions>()
+                  .end_time(),
+              7.0, 1e-5);
+}
+
 TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
   // TODO: Suport proto3 proto.Any in CalculatorOptions.
   // TODO: Avoid proto2 extensions in "RESAMPLER_OPTIONS".
@@ -473,11 +530,11 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) {
   SetUpCalculator({}, {"RESAMPLER_OPTIONS:resampler_options"}, {}, &options);
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(sequence_.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
 
-  MEDIAPIPE_EXPECT_OK(runner_->OutputSidePackets()
+  MP_EXPECT_OK(runner_->OutputSidePackets()
-                          .Tag("RESAMPLER_OPTIONS")
+                   .Tag("RESAMPLER_OPTIONS")
-                          .ValidateAsType<CalculatorOptions>());
+                   .ValidateAsType<CalculatorOptions>());
   EXPECT_NEAR(runner_->OutputSidePackets()
                   .Tag("RESAMPLER_OPTIONS")
                   .Get<CalculatorOptions>()
@@ -502,10 +559,10 @@ TEST_F(UnpackMediaSequenceCalculatorTest, GetFrameRateFromExample) {
   SetUpCalculator({}, {"IMAGE_FRAME_RATE:frame_rate"});
   runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") =
       Adopt(sequence_.release());
-  MEDIAPIPE_ASSERT_OK(runner_->Run());
+  MP_ASSERT_OK(runner_->Run());
-  MEDIAPIPE_EXPECT_OK(runner_->OutputSidePackets()
+  MP_EXPECT_OK(runner_->OutputSidePackets()
-                          .Tag("IMAGE_FRAME_RATE")
+                   .Tag("IMAGE_FRAME_RATE")
-                          .ValidateAsType<double>());
+                   .ValidateAsType<double>());
   EXPECT_EQ(runner_->OutputSidePackets().Tag("IMAGE_FRAME_RATE").Get<double>(),
             image_frame_rate_);
 }

mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc

@@ -0,0 +1,192 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iterator>
+
+#include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "tensorflow/core/example/example.pb.h"
+#include "tensorflow/core/example/feature.pb.h"
+
+namespace mediapipe {
+namespace {
+
+const char kId[] = "id";
+const char kRgb[] = "rgb";
+const char kAudio[] = "audio";
+const char kDesiredSegmentSize[] = "DESIRED_SEGMENT_SIZE";
+const char kYt8mId[] = "YT8M_ID";
+const char kYt8mSequenceExample[] = "YT8M_SEQUENCE_EXAMPLE";
+const char kQuantizedRgbFeature[] = "QUANTIZED_RGB_FEATURE";
+const char kQuantizedAudioFeature[] = "QUANTIZED_AUDIO_FEATURE";
+const char kSegmentSize[] = "SEGMENT_SIZE";
+const char kLappedTensorBufferCalculatorOptions[] =
+    "LAPPED_TENSOR_BUFFER_CALCULATOR_OPTIONS";
+
+std::string GetQuantizedFeature(
+    const tensorflow::SequenceExample& sequence_example, const std::string& key,
+    int index) {
+  const auto& bytes_list = sequence_example.feature_lists()
+                               .feature_list()
+                               .at(key)
+                               .feature()
+                               .Get(index)
+                               .bytes_list()
+                               .value();
+  CHECK_EQ(1, bytes_list.size());
+  return bytes_list.Get(0);
+}
+}  // namespace
+
+// Unpacks YT8M Sequence Example. Note that the audio feature and rgb feature
+// output are quantized. DequantizeByteArrayCalculator can do the dequantization
+// for you.
+//
+// Example config:
+// node {
+//   calculator: "UnpackYt8mSequenceExampleCalculator"
+//   input_side_packet: "YT8M_SEQUENCE_EXAMPLE:yt8m_sequence_example"
+//   output_stream: "QUANTIZED_RGB_FEATURE:quantized_rgb_feature"
+//   output_stream: "QUANTIZED_AUDIO_FEATURE:quantized_audio_feature"
+// }
+class UnpackYt8mSequenceExampleCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets()
+        .Tag(kYt8mSequenceExample)
+        .Set<tensorflow::SequenceExample>();
+    if (cc->InputSidePackets().HasTag(kDesiredSegmentSize)) {
+      cc->InputSidePackets().Tag(kDesiredSegmentSize).Set<int>();
+    }
+    cc->Outputs().Tag(kQuantizedRgbFeature).Set<std::string>();
+    cc->Outputs().Tag(kQuantizedAudioFeature).Set<std::string>();
+    if (cc->OutputSidePackets().HasTag(kYt8mId)) {
+      cc->OutputSidePackets().Tag(kYt8mId).Set<std::string>();
+    }
+    if (cc->OutputSidePackets().HasTag(kLappedTensorBufferCalculatorOptions)) {
+      cc->OutputSidePackets()
+          .Tag(kLappedTensorBufferCalculatorOptions)
+          .Set<::mediapipe::LappedTensorBufferCalculatorOptions>();
+    }
+    if (cc->OutputSidePackets().HasTag(kSegmentSize)) {
+      cc->OutputSidePackets().Tag(kSegmentSize).Set<int>();
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const tensorflow::SequenceExample& sequence_example =
+        cc->InputSidePackets()
+            .Tag(kYt8mSequenceExample)
+            .Get<tensorflow::SequenceExample>();
+    const std::string& yt8m_id =
+        sequence_example.context().feature().at(kId).bytes_list().value().Get(
+            0);
+    if (cc->OutputSidePackets().HasTag(kYt8mId)) {
+      cc->OutputSidePackets().Tag(kYt8mId).Set(
+          MakePacket<std::string>(yt8m_id));
+    }
+
+    int rgb_feature_list_length =
+        sequence_example.feature_lists().feature_list().at(kRgb).feature_size();
+    int audio_feature_list_length = sequence_example.feature_lists()
+                                        .feature_list()
+                                        .at(kAudio)
+                                        .feature_size();
+
+    if (rgb_feature_list_length != audio_feature_list_length) {
+      return ::mediapipe::FailedPreconditionError(absl::StrCat(
+          "Data corruption: the length of audio features and rgb features are "
+          "not equal. Please check the sequence example that contains yt8m "
+          "id: ",
+          yt8m_id));
+    }
+    feature_list_length_ = rgb_feature_list_length;
+    if (cc->OutputSidePackets().HasTag(kLappedTensorBufferCalculatorOptions) ||
+        cc->OutputSidePackets().HasTag(kSegmentSize)) {
+      // If the desired segment size is specified, take the min of the length of
+      // the feature list and the desired size to be the output segment size.
+      int segment_size = feature_list_length_;
+      if (cc->InputSidePackets().HasTag(kDesiredSegmentSize)) {
+        int desired_segment_size =
+            cc->InputSidePackets().Tag(kDesiredSegmentSize).Get<int>();
+        RET_CHECK(desired_segment_size > 0)
+            << "The desired segment size must be greater than zero.";
+        segment_size = std::min(
+            feature_list_length_,
+            cc->InputSidePackets().Tag(kDesiredSegmentSize).Get<int>());
+      }
+      if (cc->OutputSidePackets().HasTag(
+              kLappedTensorBufferCalculatorOptions)) {
+        auto lapped_tensor_buffer_calculator_options = absl::make_unique<
+            ::mediapipe::LappedTensorBufferCalculatorOptions>();
+        lapped_tensor_buffer_calculator_options->set_add_batch_dim_to_tensors(
+            true);
+        lapped_tensor_buffer_calculator_options->set_buffer_size(segment_size);
+        lapped_tensor_buffer_calculator_options->set_overlap(segment_size - 1);
+        lapped_tensor_buffer_calculator_options->set_timestamp_offset(
+            segment_size - 1);
+        cc->OutputSidePackets()
+            .Tag(kLappedTensorBufferCalculatorOptions)
+            .Set(Adopt(lapped_tensor_buffer_calculator_options.release()));
+      }
+      if (cc->OutputSidePackets().HasTag(kSegmentSize)) {
+        cc->OutputSidePackets()
+            .Tag(kSegmentSize)
+            .Set(MakePacket<int>(segment_size));
+      }
+    }
+    LOG(INFO) << "Reading the sequence example that contains yt8m id: "
+              << yt8m_id << ". Feature list length: " << feature_list_length_;
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (current_index_ >= feature_list_length_) {
+      return ::mediapipe::tool::StatusStop();
+    }
+    const tensorflow::SequenceExample& sequence_example =
+        cc->InputSidePackets()
+            .Tag(kYt8mSequenceExample)
+            .Get<tensorflow::SequenceExample>();
+
+    // Uses microsecond as the unit of time. In the YT8M dataset, each feature
+    // represents a second.
+    const Timestamp timestamp = Timestamp(current_index_ * 1000000);
+    cc->Outputs()
+        .Tag(kQuantizedRgbFeature)
+        .AddPacket(
+            MakePacket<std::string>(
+                GetQuantizedFeature(sequence_example, kRgb, current_index_))
+                .At(timestamp));
+    cc->Outputs()
+        .Tag(kQuantizedAudioFeature)
+        .AddPacket(
+            MakePacket<std::string>(
+                GetQuantizedFeature(sequence_example, kAudio, current_index_))
+                .At(timestamp));
+    ++current_index_;
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  int current_index_ = 0;
+  int feature_list_length_ = 0;
+};
+
+REGISTER_CALCULATOR(UnpackYt8mSequenceExampleCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc

@@ -23,10 +23,12 @@
 
 namespace mediapipe {
 
-namespace tf = ::tensorflow;
+namespace {
-
 auto& INPUT_1D = VectorFloatToTensorCalculatorOptions::INPUT_1D;
 auto& INPUT_2D = VectorFloatToTensorCalculatorOptions::INPUT_2D;
+}  // namespace
+
+namespace tf = ::tensorflow;
 
 // The calculator expects one input (a packet containing a vector<float> or
 // vector<vector<float>>) and generates one output (a packet containing a

mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc

@@ -0,0 +1,203 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Converts a single int or vector<int> or vector<vector<int>> to 1D (or 2D)
+// tf::Tensor.
+
+#include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+
+namespace mediapipe {
+
+const char kVectorInt[] = "VECTOR_INT";
+const char kSingleInt[] = "SINGLE_INT";
+const char kTensorOut[] = "TENSOR_OUT";
+
+namespace {
+auto& INPUT_1D = VectorIntToTensorCalculatorOptions::INPUT_1D;
+auto& INPUT_2D = VectorIntToTensorCalculatorOptions::INPUT_2D;
+}  // namespace
+
+namespace tf = ::tensorflow;
+
+template <typename TensorType>
+void AssignMatrixValue(int r, int c, int value, tf::Tensor* output_tensor) {
+  output_tensor->tensor<TensorType, 2>()(r, c) = value;
+}
+
+// The calculator expects one input (a packet containing a single int or
+// vector<int> or vector<vector<int>>) and generates one output (a packet
+// containing a tf::Tensor containing the same data). The output tensor will be
+// either 1D or 2D with dimensions corresponding to the input vector int. It
+// will hold DT_INT32 or DT_UINT8 or DT_INT64 values.
+//
+// Example config:
+// node {
+//  calculator: "VectorIntToTensorCalculator"
+//   input_stream: "SINGLE_INT:segment_size_int_stream"
+//   output_stream: "TENSOR_OUT:segment_size_tensor"
+// }
+//
+// or
+//
+// node {
+//   calculator: "VectorIntToTensorCalculator"
+//   input_stream: "VECTOR_INT:vector_int_features"
+//   output_stream: "TENSOR_OUT:tensor_features"
+// }
+class VectorIntToTensorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  VectorIntToTensorCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(VectorIntToTensorCalculator);
+
+::mediapipe::Status VectorIntToTensorCalculator::GetContract(
+    CalculatorContract* cc) {
+  const auto& options = cc->Options<VectorIntToTensorCalculatorOptions>();
+  // Start with only one input packet.
+  RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
+      << "Only one input stream is supported.";
+  if (options.input_size() == INPUT_2D) {
+    cc->Inputs().Tag(kVectorInt).Set<std::vector<std::vector<int>>>();
+  } else if (options.input_size() == INPUT_1D) {
+    if (cc->Inputs().HasTag(kSingleInt)) {
+      cc->Inputs().Tag(kSingleInt).Set<int>();
+    } else {
+      cc->Inputs().Tag(kVectorInt).Set<std::vector<int>>();
+    }
+  } else {
+    LOG(FATAL) << "input size not supported";
+  }
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
+      << "Only one output stream is supported.";
+  cc->Outputs().Tag(kTensorOut).Set<tf::Tensor>();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VectorIntToTensorCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<VectorIntToTensorCalculatorOptions>();
+  RET_CHECK(options_.tensor_data_type() == tf::DT_UINT8 ||
+            options_.tensor_data_type() == tf::DT_INT32 ||
+            options_.tensor_data_type() == tf::DT_INT64)
+      << "Output tensor data type is not supported.";
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status VectorIntToTensorCalculator::Process(
+    CalculatorContext* cc) {
+  tf::TensorShape tensor_shape;
+  if (options_.input_size() == INPUT_2D) {
+    const std::vector<std::vector<int>>& input =
+        cc->Inputs()
+            .Tag(kVectorInt)
+            .Value()
+            .Get<std::vector<std::vector<int>>>();
+
+    const int32 rows = input.size();
+    CHECK_GE(rows, 1);
+    const int32 cols = input[0].size();
+    CHECK_GE(cols, 1);
+    for (int i = 1; i < rows; ++i) {
+      CHECK_EQ(input[i].size(), cols);
+    }
+    if (options_.transpose()) {
+      tensor_shape = tf::TensorShape({cols, rows});
+    } else {
+      tensor_shape = tf::TensorShape({rows, cols});
+    }
+    auto output = ::absl::make_unique<tf::Tensor>(options_.tensor_data_type(),
+                                                  tensor_shape);
+    if (options_.transpose()) {
+      for (int r = 0; r < rows; ++r) {
+        for (int c = 0; c < cols; ++c) {
+          switch (options_.tensor_data_type()) {
+            case tf::DT_INT64:
+              AssignMatrixValue<tf::int64>(c, r, input[r][c], output.get());
+              break;
+            case tf::DT_UINT8:
+              AssignMatrixValue<uint8>(c, r, input[r][c], output.get());
+              break;
+            case tf::DT_INT32:
+              AssignMatrixValue<int>(c, r, input[r][c], output.get());
+              break;
+            default:
+              LOG(FATAL) << "tensor data type is not supported.";
+          }
+        }
+      }
+    } else {
+      for (int r = 0; r < rows; ++r) {
+        for (int c = 0; c < cols; ++c) {
+          switch (options_.tensor_data_type()) {
+            case tf::DT_INT64:
+              AssignMatrixValue<tf::int64>(r, c, input[r][c], output.get());
+              break;
+            case tf::DT_UINT8:
+              AssignMatrixValue<uint8>(r, c, input[r][c], output.get());
+              break;
+            case tf::DT_INT32:
+              AssignMatrixValue<int>(r, c, input[r][c], output.get());
+              break;
+            default:
+              LOG(FATAL) << "tensor data type is not supported.";
+          }
+        }
+      }
+    }
+    cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp());
+  } else if (options_.input_size() == INPUT_1D) {
+    std::vector<int> input;
+    if (cc->Inputs().HasTag(kSingleInt)) {
+      input.push_back(cc->Inputs().Tag(kSingleInt).Get<int>());
+    } else {
+      input = cc->Inputs().Tag(kVectorInt).Value().Get<std::vector<int>>();
+    }
+    CHECK_GE(input.size(), 1);
+    const int32 length = input.size();
+    tensor_shape = tf::TensorShape({length});
+    auto output = ::absl::make_unique<tf::Tensor>(options_.tensor_data_type(),
+                                                  tensor_shape);
+    for (int i = 0; i < length; ++i) {
+      switch (options_.tensor_data_type()) {
+        case tf::DT_INT64:
+          output->tensor<tf::int64, 1>()(i) = input.at(i);
+          break;
+        case tf::DT_UINT8:
+          output->tensor<uint8, 1>()(i) = input.at(i);
+          break;
+        case tf::DT_INT32:
+          output->tensor<int, 1>()(i) = input.at(i);
+          break;
+        default:
+          LOG(FATAL) << "tensor data type is not supported.";
+      }
+    }
+    cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp());
+  } else {
+    LOG(FATAL) << "input size not supported";
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto

@@ -0,0 +1,43 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "tensorflow/core/framework/types.proto";
+
+message VectorIntToTensorCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional VectorIntToTensorCalculatorOptions ext = 275364184;
+  }
+  enum InputSize {
+    UNKNOWN = 0;
+    INPUT_1D = 1;
+    INPUT_2D = 2;
+  }
+
+  // If input_size is INPUT_2D, unpack a vector<vector<int>> to a
+  // 2d tensor (matrix). If INPUT_1D, convert a single int or vector<int>
+  // into a 1d tensor (vector).
+  optional InputSize input_size = 1 [default = INPUT_1D];
+
+  // If true, the output tensor is transposed.
+  // Otherwise, the output tensor is not transposed.
+  // It will be ignored if tensor_is_2d is INPUT_1D.
+  optional bool transpose = 2 [default = false];
+
+  optional tensorflow.DataType tensor_data_type = 3 [default = DT_INT32];
+}

mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc

@@ -0,0 +1,202 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+namespace tf = ::tensorflow;
+
+class VectorIntToTensorCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUpRunner(
+      const VectorIntToTensorCalculatorOptions::InputSize input_size,
+      const tensorflow::DataType tensor_data_type, const bool transpose,
+      const bool single_value) {
+    CalculatorGraphConfig::Node config;
+    config.set_calculator("VectorIntToTensorCalculator");
+    if (single_value) {
+      config.add_input_stream("SINGLE_INT:input_int");
+    } else {
+      config.add_input_stream("VECTOR_INT:input_int");
+    }
+    config.add_output_stream("TENSOR_OUT:output_tensor");
+    auto options = config.mutable_options()->MutableExtension(
+        VectorIntToTensorCalculatorOptions::ext);
+    options->set_input_size(input_size);
+    options->set_transpose(transpose);
+    options->set_tensor_data_type(tensor_data_type);
+    runner_ = ::absl::make_unique<CalculatorRunner>(config);
+  }
+
+  void TestConvertFromVectoVectorInt(const bool transpose) {
+    SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_2D,
+                tensorflow::DT_INT32, transpose, false);
+    auto input = ::absl::make_unique<std::vector<std::vector<int>>>(
+        2, std::vector<int>(2));
+    for (int i = 0; i < 2; ++i) {
+      for (int j = 0; j < 2; ++j) {
+        input->at(i).at(j) = i * 2 + j;
+      }
+    }
+
+    const int64 time = 1234;
+    runner_->MutableInputs()
+        ->Tag("VECTOR_INT")
+        .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
+
+    EXPECT_TRUE(runner_->Run().ok());
+
+    const std::vector<Packet>& output_packets =
+        runner_->Outputs().Tag("TENSOR_OUT").packets;
+    EXPECT_EQ(1, output_packets.size());
+    EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+    const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
+
+    EXPECT_EQ(2, output_tensor.dims());
+    EXPECT_EQ(tf::DT_INT32, output_tensor.dtype());
+    const auto matrix = output_tensor.matrix<int>();
+
+    for (int i = 0; i < 2; ++i) {
+      for (int j = 0; j < 2; ++j) {
+        if (!transpose) {
+          EXPECT_EQ(i * 2 + j, matrix(i, j));
+        } else {
+          EXPECT_EQ(j * 2 + i, matrix(i, j));
+        }
+      }
+    }
+  }
+
+  std::unique_ptr<CalculatorRunner> runner_;
+};
+
+TEST_F(VectorIntToTensorCalculatorTest, TestSingleValue) {
+  SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D,
+              tensorflow::DT_INT32, false, true);
+  const int64 time = 1234;
+  runner_->MutableInputs()
+      ->Tag("SINGLE_INT")
+      .packets.push_back(MakePacket<int>(1).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("TENSOR_OUT").packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
+
+  EXPECT_EQ(1, output_tensor.dims());
+  EXPECT_EQ(tf::DT_INT32, output_tensor.dtype());
+  const auto vec = output_tensor.vec<int32>();
+  EXPECT_EQ(1, vec(0));
+}
+
+TEST_F(VectorIntToTensorCalculatorTest, TesOneDim) {
+  SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D,
+              tensorflow::DT_INT32, false, false);
+  auto input = ::absl::make_unique<std::vector<int>>(5);
+  for (int i = 0; i < 5; ++i) {
+    input->at(i) = i;
+  }
+  const int64 time = 1234;
+  runner_->MutableInputs()
+      ->Tag("VECTOR_INT")
+      .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("TENSOR_OUT").packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
+
+  EXPECT_EQ(1, output_tensor.dims());
+  EXPECT_EQ(tf::DT_INT32, output_tensor.dtype());
+  const auto vec = output_tensor.vec<int32>();
+
+  for (int i = 0; i < 5; ++i) {
+    EXPECT_EQ(i, vec(i));
+  }
+}
+
+TEST_F(VectorIntToTensorCalculatorTest, TestTwoDims) {
+  for (bool transpose : {false, true}) {
+    TestConvertFromVectoVectorInt(transpose);
+  }
+}
+
+TEST_F(VectorIntToTensorCalculatorTest, TestInt64) {
+  SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D,
+              tensorflow::DT_INT64, false, true);
+  const int64 time = 1234;
+  runner_->MutableInputs()
+      ->Tag("SINGLE_INT")
+      .packets.push_back(MakePacket<int>(2 ^ 31).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("TENSOR_OUT").packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
+
+  EXPECT_EQ(1, output_tensor.dims());
+  EXPECT_EQ(tf::DT_INT64, output_tensor.dtype());
+  const auto vec = output_tensor.vec<tf::int64>();
+  EXPECT_EQ(2 ^ 31, vec(0));
+}
+
+TEST_F(VectorIntToTensorCalculatorTest, TestUint8) {
+  SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D,
+              tensorflow::DT_UINT8, false, false);
+  auto input = ::absl::make_unique<std::vector<int>>(5);
+  for (int i = 0; i < 5; ++i) {
+    input->at(i) = i;
+  }
+  const int64 time = 1234;
+  runner_->MutableInputs()
+      ->Tag("VECTOR_INT")
+      .packets.push_back(Adopt(input.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag("TENSOR_OUT").packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const tf::Tensor& output_tensor = output_packets[0].Get<tf::Tensor>();
+
+  EXPECT_EQ(1, output_tensor.dims());
+  EXPECT_EQ(tf::DT_UINT8, output_tensor.dtype());
+  const auto vec = output_tensor.vec<uint8>();
+
+  for (int i = 0; i < 5; ++i) {
+    EXPECT_EQ(i, vec(i));
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tflite/BUILD

@@ -61,6 +61,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "tflite_tensors_to_classification_calculator_proto",
+    srcs = ["tflite_tensors_to_classification_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "tflite_tensors_to_landmarks_calculator_proto",
     srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
@@ -72,7 +79,7 @@ mediapipe_cc_proto_library(
     name = "ssd_anchors_calculator_cc_proto",
     srcs = ["ssd_anchors_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":ssd_anchors_calculator_proto"],
 )
 
@@ -80,7 +87,7 @@ mediapipe_cc_proto_library(
     name = "tflite_custom_op_resolver_calculator_cc_proto",
     srcs = ["tflite_custom_op_resolver_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_custom_op_resolver_calculator_proto"],
 )
 
@@ -88,7 +95,7 @@ mediapipe_cc_proto_library(
     name = "tflite_converter_calculator_cc_proto",
     srcs = ["tflite_converter_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_converter_calculator_proto"],
 )
 
@@ -96,7 +103,7 @@ mediapipe_cc_proto_library(
     name = "tflite_tensors_to_segmentation_calculator_cc_proto",
     srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_tensors_to_segmentation_calculator_proto"],
 )
 
@@ -104,7 +111,7 @@ mediapipe_cc_proto_library(
     name = "tflite_inference_calculator_cc_proto",
     srcs = ["tflite_inference_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_inference_calculator_proto"],
 )
 
@@ -112,15 +119,23 @@ mediapipe_cc_proto_library(
     name = "tflite_tensors_to_detections_calculator_cc_proto",
     srcs = ["tflite_tensors_to_detections_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_tensors_to_detections_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tflite_tensors_to_classification_calculator_cc_proto",
+    srcs = ["tflite_tensors_to_classification_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":tflite_tensors_to_classification_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "tflite_tensors_to_landmarks_calculator_cc_proto",
     srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":tflite_tensors_to_landmarks_calculator_proto"],
 )
 
@@ -180,12 +195,17 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "util",
+    hdrs = ["util.h"],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tflite_inference_calculator",
     srcs = ["tflite_inference_calculator.cc"],
     copts = select({
         "//mediapipe:ios": [
-            "-std=c++11",
             "-x objective-c++",
             "-fobjc-arc",  # enable reference-counting
         ],
@@ -200,6 +220,7 @@ cc_library(
     }),
     visibility = ["//visibility:public"],
     deps = [
+        ":util",
         ":tflite_inference_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/util:resource_util",
@@ -208,20 +229,25 @@ cc_library(
         "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
         "//mediapipe/framework/port:ret_check",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [
+            "//mediapipe/gpu:MPPMetalHelper",
+            "//mediapipe/gpu:MPPMetalUtil",
+            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/objc:mediapipe_framework_ios",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
+        ],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gpu_buffer",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
             "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
         ],
-        "//mediapipe:ios": [
-            "//mediapipe/gpu:MPPMetalHelper",
-            "//mediapipe/objc:mediapipe_framework_ios",
-            "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
-        ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -231,7 +257,6 @@ cc_library(
     srcs = ["tflite_converter_calculator.cc"],
     copts = select({
         "//mediapipe:ios": [
-            "-std=c++11",
             "-x objective-c++",
             "-fobjc-arc",  # enable reference-counting
         ],
@@ -246,33 +271,33 @@ cc_library(
     }),
     visibility = ["//visibility:public"],
     deps = [
+        ":util",
         ":tflite_converter_calculator_cc_proto",
         "//mediapipe/util:resource_util",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
-        "//mediapipe/framework/tool:status_util",
-        "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:ret_check",
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
-            "//mediapipe/gpu:gl_calculator_helper",
-            "//mediapipe/gpu:gpu_buffer",
-            "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
-            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
-            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
-            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
-        ],
         "//mediapipe:ios": [
+            "//mediapipe/gpu:MPPMetalUtil",
             "//mediapipe/gpu:gpu_buffer",
             "//mediapipe/gpu:MPPMetalHelper",
             "//mediapipe/objc:mediapipe_framework_ios",
             "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
         ],
-        "//conditions:default": [],
+        "//conditions:default": [
+            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gl_calculator_helper",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
+        ],
     }),
     alwayslink = 1,
 )
@@ -282,6 +307,7 @@ cc_library(
     srcs = ["tflite_tensors_to_segmentation_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":util",
         ":tflite_tensors_to_segmentation_calculator_cc_proto",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:span",
@@ -295,7 +321,9 @@ cc_library(
         "//mediapipe/util:resource_util",
         "@org_tensorflow//tensorflow/lite:framework",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gpu_buffer",
@@ -306,16 +334,49 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture",
         ],
-        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
 
+cc_test(
+    name = "tflite_tensors_to_classification_calculator_test",
+    srcs = ["tflite_tensors_to_classification_calculator_test.cc"],
+    data = ["testdata/labelmap.txt"],
+    deps = [
+        ":tflite_tensors_to_classification_calculator",
+        ":tflite_tensors_to_classification_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+)
+
 cc_library(
     name = "tflite_tensors_to_detections_calculator",
     srcs = ["tflite_tensors_to_detections_calculator.cc"],
+    copts = select({
+        "//mediapipe:ios": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
+    linkopts = select({
+        "//mediapipe:ios": [
+            "-framework CoreVideo",
+            "-framework MetalKit",
+        ],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
     deps = [
+        ":util",
         ":tflite_tensors_to_detections_calculator_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "@com_google_absl//absl/strings:str_format",
@@ -327,14 +388,52 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "@org_tensorflow//tensorflow/lite:framework",
     ] + select({
-        "//mediapipe:android": [
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [
+            "//mediapipe/gpu:MPPMetalUtil",
+            "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:MPPMetalHelper",
+            "//mediapipe/objc:mediapipe_framework_ios",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
+        ],
+        "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",
             "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader",
         ],
-        "//conditions:default": [],
+    }),
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "tflite_tensors_to_classification_calculator",
+    srcs = ["tflite_tensors_to_classification_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tflite_tensors_to_classification_calculator_cc_proto",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:location",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util:resource_util",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ] + select({
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:apple": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
     }),
     alwayslink = 1,
 )

mediapipe/calculators/tflite/ssd_anchors_calculator.cc

@@ -79,7 +79,7 @@ class SsdAnchorsCalculator : public CalculatorBase {
         cc->Options<SsdAnchorsCalculatorOptions>();
 
     auto anchors = absl::make_unique<std::vector<Anchor>>();
-    RETURN_IF_ERROR(GenerateAnchors(anchors.get(), options));
+    MP_RETURN_IF_ERROR(GenerateAnchors(anchors.get(), options));
     cc->OutputSidePackets().Index(0).Set(Adopt(anchors.release()));
     return ::mediapipe::OkStatus();
   }

mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc

@@ -90,12 +90,12 @@ TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) {
     }
   )"));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
 
   const auto& anchors =
       runner.OutputSidePackets().Index(0).Get<std::vector<Anchor>>();
   std::string anchors_string;
-  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(
+  MP_EXPECT_OK(mediapipe::file::GetContents(
       GetGoldenFilePath("anchor_golden_file_0.txt"), &anchors_string));
 
   std::vector<Anchor> anchors_golden;
@@ -133,12 +133,12 @@ TEST(SsdAnchorCalculatorTest, MobileSSDConfig) {
     }
   )"));
 
-  MEDIAPIPE_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
   const auto& anchors =
       runner.OutputSidePackets().Index(0).Get<std::vector<Anchor>>();
 
   std::string anchors_string;
-  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(
+  MP_EXPECT_OK(mediapipe::file::GetContents(
       GetGoldenFilePath("anchor_golden_file_1.txt"), &anchors_string));
 
   std::vector<Anchor> anchors_golden;

mediapipe/calculators/tflite/testdata/labelmap.txt

@@ -0,0 +1,3 @@
+classA
+classB
+classC

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -16,23 +16,24 @@
 #include <vector>
 
 #include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h"
+#include "mediapipe/calculators/tflite/util.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/matrix.h"
-#include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/resource_util.h"
 #include "tensorflow/lite/error_reporter.h"
 #include "tensorflow/lite/interpreter.h"
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
-#endif  // __ANDROID__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
 #import <CoreVideo/CoreVideo.h>
@@ -40,11 +41,13 @@
 #import <MetalKit/MetalKit.h>
 
 #import "mediapipe/gpu/MPPMetalHelper.h"
+#include "mediapipe/gpu/MPPMetalUtil.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
 #endif  // iOS
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
 typedef id<MTLBuffer> GpuTensor;
@@ -66,26 +69,28 @@ typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
 
 namespace mediapipe {
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-using ::tflite::gpu::gl::GlBuffer;
+    !defined(__APPLE__)
+using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlProgram;
 using ::tflite::gpu::gl::GlShader;
 struct GPUData {
   int elements = 1;
-  GlBuffer buffer;
+  GpuTensor buffer;
   GlShader shader;
   GlProgram program;
 };
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
 struct GPUData {
   int elements = 1;
-  id<MTLBuffer> buffer;
+  GpuTensor buffer;
   id<MTLComputePipelineState> pipeline_state;
 };
 #endif
 
 // Calculator for normalizing and converting an ImageFrame or Matrix
-// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer.
+// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
+// or MTLBuffer.
 //
 // This calculator is designed to be used with the TfLiteInferenceCalcualtor,
 // as a pre-processing step for calculator inputs.
@@ -102,7 +107,7 @@ struct GPUData {
 // Output:
 //  One of the following tags:
 //  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32, or kTfLiteUint8.
-//  TENSORS_GPU - vector of GlBuffer.
+//  TENSORS_GPU - vector of GlBuffer or MTLBuffer.
 //
 // Example use:
 // node {
@@ -144,7 +149,8 @@ class TfLiteConverterCalculator : public CalculatorBase {
 
   std::unique_ptr<tflite::Interpreter> interpreter_ = nullptr;
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GPUData> gpu_data_out_;
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
@@ -175,25 +181,34 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
             cc->Outputs().HasTag("TENSORS_GPU"));
 
+  bool use_gpu = false;
+
   if (cc->Inputs().HasTag("IMAGE")) cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
   if (cc->Inputs().HasTag("MATRIX")) cc->Inputs().Tag("MATRIX").Set<Matrix>();
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("IMAGE_GPU"))
+  if (cc->Inputs().HasTag("IMAGE_GPU")) {
     cc->Inputs().Tag("IMAGE_GPU").Set<mediapipe::GpuBuffer>();
-#endif
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   if (cc->Outputs().HasTag("TENSORS"))
     cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU"))
+  if (cc->Outputs().HasTag("TENSORS_GPU")) {
     cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
-#endif
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
-#if defined(__ANDROID__)
+  if (use_gpu) {
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
-  RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
+    MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif
+  }
 
   // Assign this calculator's default InputStreamHandler.
   cc->SetInputStreamHandler("FixedSizeInputStreamHandler");
@@ -204,14 +219,14 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 ::mediapipe::Status TfLiteConverterCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  RETURN_IF_ERROR(LoadOptions(cc));
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
 
   if (cc->Inputs().HasTag("IMAGE_GPU") ||
       cc->Outputs().HasTag("IMAGE_OUT_GPU")) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     use_gpu_ = true;
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif
   }
 
@@ -221,8 +236,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
               cc->Outputs().HasTag("TENSORS_GPU"));
     // Cannot use quantization.
     use_quantized_tensors_ = false;
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
     gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
     RET_CHECK(gpu_helper_);
@@ -238,22 +254,24 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) {
   if (use_gpu_) {
+    // GpuBuffer to tflite::gpu::GlBuffer conversion.
     if (!initialized_) {
-      RETURN_IF_ERROR(InitGpu(cc));
+      MP_RETURN_IF_ERROR(InitGpu(cc));
       initialized_ = true;
     }
     // Convert to GPU tensors type.
-    RETURN_IF_ERROR(ProcessGPU(cc));
+    MP_RETURN_IF_ERROR(ProcessGPU(cc));
   } else {
     // Convert to CPU tensors or Matrix type.
-    RETURN_IF_ERROR(ProcessCPU(cc));
+    MP_RETURN_IF_ERROR(ProcessCPU(cc));
   }
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::Close(CalculatorContext* cc) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
   gpu_helper_.RunInGlContext([this] { gpu_data_out_.reset(); });
 #endif
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
@@ -321,11 +339,11 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
       float* tensor_buffer = tensor->data.f;
       RET_CHECK(tensor_buffer);
       if (image_frame.ByteDepth() == 1) {
-        RETURN_IF_ERROR(NormalizeImage<uint8>(image_frame, zero_center_,
+        MP_RETURN_IF_ERROR(NormalizeImage<uint8>(
-                                              flip_vertically_, tensor_buffer));
+            image_frame, zero_center_, flip_vertically_, tensor_buffer));
       } else if (image_frame.ByteDepth() == 4) {
-        RETURN_IF_ERROR(NormalizeImage<float>(image_frame, zero_center_,
+        MP_RETURN_IF_ERROR(NormalizeImage<float>(
-                                              flip_vertically_, tensor_buffer));
+            image_frame, zero_center_, flip_vertically_, tensor_buffer));
       } else {
         return ::mediapipe::InternalError(
             "Only byte-based (8 bit) and float (32 bit) images supported.");
@@ -359,7 +377,7 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
     float* tensor_buffer = tensor->data.f;
     RET_CHECK(tensor_buffer);
 
-    RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_buffer));
+    MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_buffer));
 
     auto output_tensors = absl::make_unique<std::vector<TfLiteTensor>>();
     output_tensors->emplace_back(*tensor);
@@ -372,26 +390,21 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
 ::mediapipe::Status TfLiteConverterCalculator::ProcessGPU(
     CalculatorContext* cc) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
   // GpuBuffer to tflite::gpu::GlBuffer conversion.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
-  RETURN_IF_ERROR(
+  MP_RETURN_IF_ERROR(
       gpu_helper_.RunInGlContext([this, &input]() -> ::mediapipe::Status {
         // Convert GL texture into TfLite GlBuffer (SSBO).
         auto src = gpu_helper_.CreateSourceTexture(input);
         glActiveTexture(GL_TEXTURE0 + 0);
         glBindTexture(GL_TEXTURE_2D, src.name());
-        auto status = gpu_data_out_->buffer.BindToIndex(1);
+        RET_CHECK_CALL(gpu_data_out_->buffer.BindToIndex(1));
-        if (!status.ok()) {
-          return ::mediapipe::InternalError(status.error_message());
-        }
         const tflite::gpu::uint3 workgroups = {
             NumGroups(input.width(), kWorkgroupSize),
             NumGroups(input.height(), kWorkgroupSize), 1};
-        status = gpu_data_out_->program.Dispatch(workgroups);
+        RET_CHECK_CALL(gpu_data_out_->program.Dispatch(workgroups));
-        if (!status.ok()) {
-          return ::mediapipe::InternalError(status.error_message());
-        }
         glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);
         glBindTexture(GL_TEXTURE_2D, 0);
         src.Release();
@@ -400,17 +413,17 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   // Copy into outputs.
   auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
-  output_tensors->resize(1);
+  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-  {
+      [this, &output_tensors]() -> ::mediapipe::Status {
-    GlBuffer& tensor = output_tensors->at(0);
+        output_tensors->resize(1);
-    using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
+        {
-    auto status = CreateReadWriteShaderStorageBuffer<float>(
+          GpuTensor& tensor = output_tensors->at(0);
-        gpu_data_out_->elements, &tensor);
+          RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
-    if (!status.ok()) {
+              gpu_data_out_->elements, &tensor));
-      return ::mediapipe::InternalError(status.error_message());
+          RET_CHECK_CALL(CopyBuffer(gpu_data_out_->buffer, tensor));
-    }
+        }
-    tflite::gpu::gl::CopyBuffer(gpu_data_out_->buffer, tensor);
+        return ::mediapipe::OkStatus();
-  }
+      }));
   cc->Outputs()
       .Tag("TENSORS_GPU")
       .Add(output_tensors.release(), cc->InputTimestamp());
@@ -438,66 +451,61 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   }
 
   // Copy into outputs.
+  // TODO Avoid this copy.
   auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
+  output_tensors->resize(1);
   {
     id<MTLDevice> device = gpu_helper_.mtlDevice;
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
+    output_tensors->at(0) =
-    command_buffer.label = @"TfLiteConverterCalculatorCopy";
-    id<MTLBuffer> tensor =
         [device newBufferWithLength:gpu_data_out_->elements * sizeof(float)
                             options:MTLResourceStorageModeShared];
-    id<MTLBlitCommandEncoder> blit_command =
+    [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0)
-        [command_buffer blitCommandEncoder];
+                               from:gpu_data_out_->buffer
-    [blit_command copyFromBuffer:gpu_data_out_->buffer
+                           blocking:true
-                    sourceOffset:0
+                      commandBuffer:[gpu_helper_ commandBuffer]];
-                        toBuffer:tensor
-               destinationOffset:0
-                            size:gpu_data_out_->elements * sizeof(float)];
-    [blit_command endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
-
-    output_tensors->push_back(tensor);
   }
 
   cc->Outputs()
       .Tag("TENSORS_GPU")
       .Add(output_tensors.release(), cc->InputTimestamp());
 #else
-  RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+  RET_CHECK_FAIL() << "GPU processing is not enabled.";
 #endif
 
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status TfLiteConverterCalculator::InitGpu(CalculatorContext* cc) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  // Configure inputs.
+  // Get input image sizes.
   const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
   mediapipe::ImageFormat::Format format =
       mediapipe::ImageFormatForGpuBufferFormat(input.format());
   gpu_data_out_ = absl::make_unique<GPUData>();
   gpu_data_out_->elements = input.height() * input.width() * max_num_channels_;
   const bool include_alpha = (max_num_channels_ == 4);
-  if (!(format == mediapipe::ImageFormat::SRGB ||
+  const bool single_channel = (max_num_channels_ == 1);
+  if (!(format == mediapipe::ImageFormat::GRAY8 ||
+        format == mediapipe::ImageFormat::SRGB ||
         format == mediapipe::ImageFormat::SRGBA))
     RET_CHECK_FAIL() << "Unsupported GPU input format.";
   if (include_alpha && (format != mediapipe::ImageFormat::SRGBA))
     RET_CHECK_FAIL() << "Num input channels is less than desired output.";
-#endif
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-  // Device memory.
+    !defined(__APPLE__)
-  auto status = ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-      gpu_data_out_->elements, &gpu_data_out_->buffer);
+      [this, &include_alpha, &input, &single_channel]() -> ::mediapipe::Status {
-  if (!status.ok()) {
+        // Device memory.
-    return ::mediapipe::InternalError(status.error_message());
+        RET_CHECK_CALL(
-  }
+            ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+                gpu_data_out_->elements, &gpu_data_out_->buffer));
 
-  // Shader to convert GL Texture to Shader Storage Buffer Object (SSBO),
+        // Shader to convert GL Texture to Shader Storage Buffer Object (SSBO),
-  // with normalization to either: [0,1] or [-1,1].
+        // with normalization to either: [0,1] or [-1,1].
-  const std::string shader_source = absl::Substitute(
+        const std::string shader_source = absl::Substitute(
-      R"( #version 310 es
+            R"( #version 310 es
           layout(local_size_x = $0, local_size_y = $0) in;
           layout(binding = 0) uniform sampler2D input_texture;
           layout(std430, binding = 1) buffer Output {float elements[];} output_data;
@@ -505,33 +513,31 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
           void main() {
             ivec2 gid = ivec2(gl_GlobalInvocationID.xy);
             if (gid.x >= width_height.x || gid.y >= width_height.y) return;
-            $5  // pixel fetch
+            vec4 pixel = texelFetch(input_texture, gid, 0);
             $3  // normalize [-1,1]
             int linear_index = $7 * ($4 * width_height.x + gid.x);
-            output_data.elements[linear_index + 0] = pixel.x;
+            output_data.elements[linear_index + 0] = pixel.x;  // r channel
-            output_data.elements[linear_index + 1] = pixel.y;
+            $5  // g & b channels
-            output_data.elements[linear_index + 2] = pixel.z;
             $6  // alpha channel
           })",
-      /*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(),
+            /*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(),
-      /*$3=*/zero_center_ ? "pixel = (pixel - 0.5) * 2.0;" : "",
+            /*$3=*/zero_center_ ? "pixel = (pixel - 0.5) * 2.0;" : "",
-      /*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y",
+            /*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y",
-      /*$5=*/
+            /*$5=*/
-      include_alpha ? "vec4 pixel = texelFetch(input_texture, gid, 0);"
+            single_channel
-                    : "vec3 pixel = texelFetch(input_texture, gid, 0).xyz;",
+                ? ""
-      /*$6=*/
+                : R"(output_data.elements[linear_index + 1] = pixel.y;
-      include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;" : "",
+                            output_data.elements[linear_index + 2] = pixel.z;)",
-      /*$7=*/include_alpha ? 4 : 3);
+            /*$6=*/
-  status = GlShader::CompileShader(GL_COMPUTE_SHADER, shader_source,
+            include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;"
-                                   &gpu_data_out_->shader);
+                          : "",
-  if (!status.ok()) {
+            /*$7=*/max_num_channels_);
-    return ::mediapipe::InternalError(status.error_message());
+        RET_CHECK_CALL(GlShader::CompileShader(GL_COMPUTE_SHADER, shader_source,
-  }
+                                               &gpu_data_out_->shader));
-  status = GlProgram::CreateWithShader(gpu_data_out_->shader,
+        RET_CHECK_CALL(GlProgram::CreateWithShader(gpu_data_out_->shader,
-                                       &gpu_data_out_->program);
+                                                   &gpu_data_out_->program));
-  if (!status.ok()) {
+        return ::mediapipe::OkStatus();
-    return ::mediapipe::InternalError(status.error_message());
+      }));
-  }
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
   RET_CHECK(include_alpha)
       << "iOS GPU inference currently accepts only RGBA input.";
@@ -546,8 +552,6 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
   // with normalization to either: [0,1] or [-1,1].
   const std::string shader_source = absl::Substitute(
       R"(
-  #include <simd/simd.h>
-
   #include <metal_stdlib>
 
   using namespace metal;
@@ -612,9 +616,9 @@ REGISTER_CALCULATOR(TfLiteConverterCalculator);
 
   // Get desired way to handle input channels.
   max_num_channels_ = options.max_num_channels();
-  // Currently only alpha channel toggling is suppored.
+  CHECK_GE(max_num_channels_, 1);
-  CHECK_GE(max_num_channels_, 3);
   CHECK_LE(max_num_channels_, 4);
+  CHECK_NE(max_num_channels_, 2);
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
   if (cc->Inputs().HasTag("IMAGE_GPU"))
     // Currently on iOS, tflite gpu input tensor must be 4 channels,

mediapipe/calculators/tflite/tflite_converter_calculator.proto

@@ -36,8 +36,7 @@ message TfLiteConverterCalculatorOptions {
   optional bool flip_vertically = 2 [default = false];
 
   // Controls how many channels of the input image get passed through to the
-  // tensor. Currently this only controls whether or not to ignore alpha
+  // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU.
-  // channel, so it must be 3 or 4.
   optional int32 max_num_channels = 3 [default = 3];
 
   // The calculator expects Matrix inputs to be in column-major order. Set

mediapipe/calculators/tflite/tflite_converter_calculator_test.cc

@@ -67,7 +67,7 @@ class TfLiteConverterCalculatorTest : public ::testing::Test {
         }
       }
     }
-    MEDIAPIPE_ASSERT_OK(graph_->AddPacketToInputStream(
+    MP_ASSERT_OK(graph_->AddPacketToInputStream(
         "matrix", Adopt(matrix.release()).At(Timestamp(0))));
   }
 
@@ -99,14 +99,14 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
 
     // Run the graph.
     graph_ = absl::make_unique<CalculatorGraph>();
-    MEDIAPIPE_ASSERT_OK(graph_->Initialize(graph_config));
+    MP_ASSERT_OK(graph_->Initialize(graph_config));
-    MEDIAPIPE_ASSERT_OK(graph_->StartRun({}));
+    MP_ASSERT_OK(graph_->StartRun({}));
 
     // Push the tensor into the graph.
     AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/false);
 
     // Wait until the calculator done processing.
-    MEDIAPIPE_ASSERT_OK(graph_->WaitUntilIdle());
+    MP_ASSERT_OK(graph_->WaitUntilIdle());
     EXPECT_EQ(1, output_packets.size());
 
     // Get and process results.
@@ -128,8 +128,8 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) {
 
     // Fully close graph at end, otherwise calculator+tensors are destroyed
     // after calling WaitUntilDone().
-    MEDIAPIPE_ASSERT_OK(graph_->CloseInputStream("matrix"));
+    MP_ASSERT_OK(graph_->CloseInputStream("matrix"));
-    MEDIAPIPE_ASSERT_OK(graph_->WaitUntilDone());
+    MP_ASSERT_OK(graph_->WaitUntilDone());
 
     graph_.reset();
   }
@@ -160,14 +160,14 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
 
     // Run the graph.
     graph_ = absl::make_unique<CalculatorGraph>();
-    MEDIAPIPE_ASSERT_OK(graph_->Initialize(graph_config));
+    MP_ASSERT_OK(graph_->Initialize(graph_config));
-    MEDIAPIPE_ASSERT_OK(graph_->StartRun({}));
+    MP_ASSERT_OK(graph_->StartRun({}));
 
     // Push the tensor into the graph.
     AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/true);
 
     // Wait until the calculator done processing.
-    MEDIAPIPE_ASSERT_OK(graph_->WaitUntilIdle());
+    MP_ASSERT_OK(graph_->WaitUntilIdle());
     EXPECT_EQ(1, output_packets.size());
 
     // Get and process results.
@@ -189,8 +189,8 @@ TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) {
 
     // Fully close graph at end, otherwise calculator+tensors are destroyed
     // after calling WaitUntilDone().
-    MEDIAPIPE_ASSERT_OK(graph_->CloseInputStream("matrix"));
+    MP_ASSERT_OK(graph_->CloseInputStream("matrix"));
-    MEDIAPIPE_ASSERT_OK(graph_->WaitUntilDone());
+    MP_ASSERT_OK(graph_->WaitUntilDone());
 
     graph_.reset();
   }

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -12,10 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstring>
+#include <memory>
 #include <string>
 #include <vector>
 
 #include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
+#include "mediapipe/calculators/tflite/util.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/resource_util.h"
@@ -24,14 +27,16 @@
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
+#include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_program.h"
 #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
-#endif  // __ANDROID__
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
 #import <CoreVideo/CoreVideo.h>
@@ -39,33 +44,44 @@
 #import <MetalKit/MetalKit.h>
 
 #import "mediapipe/gpu/MPPMetalHelper.h"
+#include "mediapipe/gpu/MPPMetalUtil.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "tensorflow/lite/delegates/gpu/common/shape.h"
+#include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h"
 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
 #endif  // iOS
 
-#if defined(__ANDROID__)
+namespace {
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
 typedef ::tflite::gpu::gl::GlBuffer GpuTensor;
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
 typedef id<MTLBuffer> GpuTensor;
 #endif
 
+// Round up n to next multiple of m.
+size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
+}  // namespace
+
 // TfLiteInferenceCalculator File Layout:
 //  * Header
 //  * Core
 //  * Aux
 namespace mediapipe {
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
+using ::tflite::gpu::gl::CopyBuffer;
+using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
 using ::tflite::gpu::gl::GlBuffer;
-using ::tflite::gpu::gl::GlProgram;
+#endif
-using ::tflite::gpu::gl::GlShader;
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
 struct GPUData {
   int elements = 1;
-  GlBuffer buffer;
+  GpuTensor buffer;
-};
+  ::tflite::gpu::BHWC shape;
-#elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
-struct GPUData {
-  int elements = 1;
-  id<MTLBuffer> buffer;
 };
 #endif
 
@@ -134,7 +150,8 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   std::unique_ptr<tflite::FlatBufferModel> model_;
   TfLiteDelegate* delegate_ = nullptr;
 
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
   mediapipe::GlCalculatorHelper gpu_helper_;
   std::unique_ptr<GPUData> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
@@ -142,6 +159,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   MPPMetalHelper* gpu_helper_ = nullptr;
   std::unique_ptr<GPUData> gpu_data_in_;
   std::vector<std::unique_ptr<GPUData>> gpu_data_out_;
+  TFLBufferConvert* converter_from_BPHWC4_ = nil;
 #endif
 
   std::string model_path_ = "";
@@ -161,19 +179,25 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
             cc->Outputs().HasTag("TENSORS_GPU"));
 
+  bool use_gpu = false;
+
   if (cc->Inputs().HasTag("TENSORS"))
     cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Inputs().HasTag("TENSORS_GPU"))
+  if (cc->Inputs().HasTag("TENSORS_GPU")) {
     cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
-#endif
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   if (cc->Outputs().HasTag("TENSORS"))
     cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
-  if (cc->Outputs().HasTag("TENSORS_GPU"))
+  if (cc->Outputs().HasTag("TENSORS_GPU")) {
     cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
-#endif
+    use_gpu |= true;
+  }
+#endif  //  !MEDIAPIPE_DISABLE_GPU
 
   if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
     cc->InputSidePackets()
@@ -181,11 +205,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         .Set<tflite::ops::builtin::BuiltinOpResolver>();
   }
 
-#if defined(__ANDROID__)
+  const auto& options =
-  RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+      cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
+  use_gpu |= options.use_gpu();
+
+  if (use_gpu) {
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
-  RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
+    MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif
+  }
 
   // Assign this calculator's default InputStreamHandler.
   cc->SetInputStreamHandler("FixedSizeInputStreamHandler");
@@ -196,40 +227,52 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 ::mediapipe::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) {
   cc->SetOffset(TimestampDiff(0));
 
-  RETURN_IF_ERROR(LoadOptions(cc));
+  MP_RETURN_IF_ERROR(LoadOptions(cc));
 
   if (cc->Inputs().HasTag("TENSORS_GPU")) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_input_ = true;
     gpu_inference_ = true;  // Inference must be on GPU also.
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
-#endif
+        << "GPU processing not enabled.";
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
   if (cc->Outputs().HasTag("TENSORS_GPU")) {
-#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
     gpu_output_ = true;
     RET_CHECK(cc->Inputs().HasTag("TENSORS_GPU"))
         << "GPU output must also have GPU Input.";
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK(!cc->Inputs().HasTag("TENSORS_GPU"))
-#endif
+        << "GPU processing not enabled.";
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
-  RETURN_IF_ERROR(LoadModel(cc));
+  MP_RETURN_IF_ERROR(LoadModel(cc));
 
   if (gpu_inference_) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    RETURN_IF_ERROR(gpu_helper_.Open(cc));
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
     gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
     RET_CHECK(gpu_helper_);
 #endif
-
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    RETURN_IF_ERROR(LoadDelegate(cc));
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+        [this, &cc]() -> ::mediapipe::Status { return LoadDelegate(cc); }));
+#else
+    MP_RETURN_IF_ERROR(LoadDelegate(cc));
+#endif
   }
 
+#if defined(__EMSCRIPTEN__)
+  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+#endif  // __EMSCRIPTEN__
+
   return ::mediapipe::OkStatus();
 }
 
@@ -237,35 +280,28 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   // 1. Receive pre-processed tensor inputs.
   if (gpu_input_) {
     // Read GPU input into SSBO.
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
     RET_CHECK_EQ(input_tensors.size(), 1);
-    RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
         [this, &input_tensors]() -> ::mediapipe::Status {
           // Explicit copy input.
-          tflite::gpu::gl::CopyBuffer(input_tensors[0], gpu_data_in_->buffer);
+          RET_CHECK_CALL(CopyBuffer(input_tensors[0], gpu_data_in_->buffer));
           return ::mediapipe::OkStatus();
         }));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
     const auto& input_tensors =
         cc->Inputs().Tag("TENSORS_GPU").Get<std::vector<GpuTensor>>();
     RET_CHECK_EQ(input_tensors.size(), 1);
-    id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    command_buffer.label = @"TfLiteInferenceCalculatorInput";
-    id<MTLBlitCommandEncoder> blit_command =
-        [command_buffer blitCommandEncoder];
     // Explicit copy input.
-    [blit_command copyFromBuffer:input_tensors[0]
+    [MPPMetalUtil blitMetalBufferTo:gpu_data_in_->buffer
-                    sourceOffset:0
+                               from:input_tensors[0]
-                        toBuffer:gpu_data_in_->buffer
+                           blocking:true
-               destinationOffset:0
+                      commandBuffer:[gpu_helper_ commandBuffer]];
-                            size:gpu_data_in_->elements * sizeof(float)];
-    [blit_command endEncoding];
-    [command_buffer commit];
-    [command_buffer waitUntilCompleted];
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
 #endif
   } else {
     // Read CPU input into tensors.
@@ -278,22 +314,26 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       if (use_quantized_tensors_) {
         const uint8* input_tensor_buffer = input_tensor->data.uint8;
         uint8* local_tensor_buffer = interpreter_->typed_input_tensor<uint8>(i);
-        memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor->bytes);
+        std::memcpy(local_tensor_buffer, input_tensor_buffer,
+                    input_tensor->bytes);
       } else {
         const float* input_tensor_buffer = input_tensor->data.f;
         float* local_tensor_buffer = interpreter_->typed_input_tensor<float>(i);
-        memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor->bytes);
+        std::memcpy(local_tensor_buffer, input_tensor_buffer,
+                    input_tensor->bytes);
       }
     }
   }
 
   // 2. Run inference.
   if (gpu_inference_) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
+    !defined(__APPLE__)
-      RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+    MP_RETURN_IF_ERROR(
-      return ::mediapipe::OkStatus();
+        gpu_helper_.RunInGlContext([this]() -> ::mediapipe::Status {
-    }));
+          RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+          return ::mediapipe::OkStatus();
+        }));
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
     RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
 #endif
@@ -303,52 +343,52 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // 3. Output processed tensors.
   if (gpu_output_) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
-    output_tensors->resize(gpu_data_out_.size());
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-    for (int i = 0; i < gpu_data_out_.size(); ++i) {
+        [this, &output_tensors]() -> ::mediapipe::Status {
-      GlBuffer& tensor = output_tensors->at(i);
+          output_tensors->resize(gpu_data_out_.size());
-      using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
+          for (int i = 0; i < gpu_data_out_.size(); ++i) {
-      auto status = CreateReadWriteShaderStorageBuffer<float>(
+            GpuTensor& tensor = output_tensors->at(i);
-          gpu_data_out_[i]->elements, &tensor);
+            RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
-      if (!status.ok()) {
+                gpu_data_out_[i]->elements, &tensor));
-        return ::mediapipe::InternalError(status.error_message());
+            RET_CHECK_CALL(CopyBuffer(gpu_data_out_[i]->buffer, tensor));
-      }
+          }
-      tflite::gpu::gl::CopyBuffer(gpu_data_out_[i]->buffer, tensor);
+          return ::mediapipe::OkStatus();
-    }
+        }));
     cc->Outputs()
         .Tag("TENSORS_GPU")
         .Add(output_tensors.release(), cc->InputTimestamp());
 #elif defined(__APPLE__) && !TARGET_OS_OSX  // iOS
     // Output result tensors (GPU).
     auto output_tensors = absl::make_unique<std::vector<GpuTensor>>();
+    output_tensors->resize(gpu_data_out_.size());
     id<MTLDevice> device = gpu_helper_.mtlDevice;
     id<MTLCommandBuffer> command_buffer = [gpu_helper_ commandBuffer];
-    command_buffer.label = @"TfLiteInferenceCalculatorOutput";
+    command_buffer.label = @"TfLiteInferenceBPHWC4Convert";
+    id<MTLComputeCommandEncoder> convert_command =
+        [command_buffer computeCommandEncoder];
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
-      id<MTLBuffer> tensor =
+      output_tensors->at(i) =
           [device newBufferWithLength:gpu_data_out_[i]->elements * sizeof(float)
                               options:MTLResourceStorageModeShared];
-      id<MTLBlitCommandEncoder> blit_command =
+      // Reshape tensor.
-          [command_buffer blitCommandEncoder];
+      [converter_from_BPHWC4_ convertWithEncoder:convert_command
-      // Explicit copy input.
+                                           shape:gpu_data_out_[i]->shape
-      [blit_command copyFromBuffer:gpu_data_out_[i]->buffer
+                                    sourceBuffer:gpu_data_out_[i]->buffer
-                      sourceOffset:0
+                                 convertedBuffer:output_tensors->at(i)];
-                          toBuffer:tensor
-                 destinationOffset:0
-                              size:gpu_data_out_[i]->elements * sizeof(float)];
-      [blit_command endEncoding];
-      [command_buffer commit];
-      [command_buffer waitUntilCompleted];
-      output_tensors->push_back(tensor);
     }
+    [convert_command endEncoding];
+    [command_buffer commit];
+    [command_buffer waitUntilCompleted];
     cc->Outputs()
         .Tag("TENSORS_GPU")
         .Add(output_tensors.release(), cc->InputTimestamp());
 #else
-    RET_CHECK_FAIL() << "GPU processing is for Android and iOS only.";
+    RET_CHECK_FAIL() << "GPU processing not enabled.";
-#endif
+#endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
     // Output result tensors (CPU).
     const auto& tensor_indexes = interpreter_->outputs();
@@ -366,8 +406,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) {
   if (delegate_) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
-    RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
+    !defined(__APPLE__)
+    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
       TfLiteGpuDelegateDelete(delegate_);
       gpu_data_in_.reset();
       for (int i = 0; i < gpu_data_out_.size(); ++i) {
@@ -430,12 +471,17 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   RET_CHECK(interpreter_);
 
+#if defined(__EMSCRIPTEN__)
+  interpreter_->SetNumThreads(1);
+#endif  // __EMSCRIPTEN__
+
   if (gpu_output_) {
     use_quantized_tensors_ = false;
   } else {
     RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
-    use_quantized_tensors_ = (interpreter_->tensor(0)->quantization.type ==
+    use_quantized_tensors_ =
-                              kTfLiteAffineQuantization);
+        (interpreter_->tensor(interpreter_->inputs()[0])->quantization.type ==
+         kTfLiteAffineQuantization);
     if (use_quantized_tensors_) gpu_inference_ = false;
   }
 
@@ -444,7 +490,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
     CalculatorContext* cc) {
-#if defined(__ANDROID__)
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+    !defined(__APPLE__)
   // Configure and create the delegate.
   TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
   options.compile_options.precision_loss_allowed = 1;
@@ -464,15 +511,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     for (int d = 0; d < tensor->dims->size; ++d) {
       gpu_data_in_->elements *= tensor->dims->data[d];
     }
-    // Input to model can be either RGB/RGBA only.
+    CHECK_GE(tensor->dims->data[3], 1);
-    RET_CHECK_GE(tensor->dims->data[3], 3);
+    CHECK_LE(tensor->dims->data[3], 4);
-    RET_CHECK_LE(tensor->dims->data[3], 4);
+    CHECK_NE(tensor->dims->data[3], 2);
     // Create and bind input buffer.
-    auto status = ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
+    RET_CHECK_CALL(::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
-        gpu_data_in_->elements, &gpu_data_in_->buffer);
+        gpu_data_in_->elements, &gpu_data_in_->buffer));
-    if (!status.ok()) {
-      return ::mediapipe::InternalError(status.error_message());
-    }
     RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
                      delegate_, gpu_data_in_->buffer.id(),
                      interpreter_->inputs()[0]),  // First tensor only
@@ -494,12 +538,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     // Create and bind output buffers.
     interpreter_->SetAllowBufferHandleOutput(true);
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
-      using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer;
+      RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
-      auto status = CreateReadWriteShaderStorageBuffer<float>(
+          gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
-          gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer);
-      if (!status.ok()) {
-        return ::mediapipe::InternalError(status.error_message());
-      }
       RET_CHECK_EQ(
           TfLiteGpuDelegateBindBufferToTensor(
               delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
@@ -509,14 +549,15 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   // Must call this last.
   RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
-#endif  // __ANDROID__
+#endif  // OpenGL
 
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
   // Configure and create the delegate.
   GpuDelegateOptions options;
   options.allow_precision_loss = false;  // Must match converter, F=float/T=half
-  options.wait_type = GpuDelegateOptions::WaitType::kActive;
+  options.wait_type = GpuDelegateOptions::WaitType::kPassive;
   if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
+  id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   if (gpu_input_) {
     // Get input image sizes.
@@ -537,11 +578,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
     }
     // Create and bind input buffer.
-    id<MTLDevice> device = gpu_helper_.mtlDevice;
     gpu_data_in_->buffer =
         [device newBufferWithLength:gpu_data_in_->elements * sizeof(float)
                             options:MTLResourceStorageModeShared];
-    // Must call this before TFLGpuDelegateBindMetalBufferToTensor.
     RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
     RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
                      delegate_,
@@ -559,12 +598,33 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       gpu_data_out_[i]->elements = 1;
       // TODO handle *2 properly on some dialated models
       for (int d = 0; d < tensor->dims->size; ++d) {
-        gpu_data_out_[i]->elements *= tensor->dims->data[d];
+        // Pad each dim for BHWC4 conversion inside delegate.
+        gpu_data_out_[i]->elements *= RoundUp(tensor->dims->data[d], 4);
+      }
+      // Save dimensions for reshaping back later.
+      gpu_data_out_[i]->shape.b = tensor->dims->data[0];
+      switch (tensor->dims->size) {
+        case 2:
+          gpu_data_out_[i]->shape.h = 1;
+          gpu_data_out_[i]->shape.w = 1;
+          gpu_data_out_[i]->shape.c = tensor->dims->data[1];
+          break;
+        case 3:
+          gpu_data_out_[i]->shape.h = 1;
+          gpu_data_out_[i]->shape.w = tensor->dims->data[1];
+          gpu_data_out_[i]->shape.c = tensor->dims->data[2];
+          break;
+        case 4:
+          gpu_data_out_[i]->shape.h = tensor->dims->data[1];
+          gpu_data_out_[i]->shape.w = tensor->dims->data[2];
+          gpu_data_out_[i]->shape.c = tensor->dims->data[3];
+          break;
+        default:
+          return mediapipe::InternalError("Unsupported tensor shape.");
       }
     }
     // Create and bind output buffers.
     interpreter_->SetAllowBufferHandleOutput(true);
-    id<MTLDevice> device = gpu_helper_.mtlDevice;
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
       gpu_data_out_[i]->buffer =
           [device newBufferWithLength:gpu_data_out_[i]->elements * sizeof(float)
@@ -573,6 +633,14 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
                        delegate_, output_indices[i], gpu_data_out_[i]->buffer),
                    true);
     }
+    // Create converter for GPU output.
+    converter_from_BPHWC4_ = [[TFLBufferConvert alloc] initWithDevice:device
+                                                            isFloat16:false
+                                                      convertToPBHWC4:false];
+    if (converter_from_BPHWC4_ == nil) {
+      return mediapipe::InternalError(
+          "Error initializating output buffer converter");
+    }
   }
 #endif  // iOS
 

mediapipe/calculators/tflite/tflite_inference_calculator_test.cc

@@ -93,13 +93,13 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
   CalculatorGraph graph(graph_config);
-  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.StartRun({}));
 
   // Push the tensor into the graph.
-  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
       "tensor_in", Adopt(input_vec.release()).At(Timestamp(0))));
   // Wait until the calculator done processing.
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilIdle());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
   ASSERT_EQ(1, output_packets.size());
 
   // Get and process results.
@@ -116,8 +116,8 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
 
   // Fully close graph at end, otherwise calculator+tensors are destroyed
   // after calling WaitUntilDone().
-  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("tensor_in"));
+  MP_ASSERT_OK(graph.CloseInputStream("tensor_in"));
-  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+  MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
 }  // namespace mediapipe

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc

@@ -0,0 +1,180 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <unordered_map>
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "absl/types/span.h"
+#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+#include "tensorflow/lite/interpreter.h"
+#if defined(__EMSCRIPTEN__) || defined(__ANDROID__) || \
+    (defined(__APPLE__) && !TARGET_OS_OSX)
+#include "mediapipe/util/android/file/base/file.h"
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+// Convert result TFLite tensors from classification models into MediaPipe
+// classifications.
+//
+// Input:
+//  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 containing one
+//            tensor, the size of which must be (1, * num_classes).
+// Output:
+//  CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index
+//                    fields of each classification are set, while the label
+//                    field is only set if label_map_path is provided.
+//
+// Usage example:
+// node {
+//   calculator: "TfLiteTensorsToClassificationCalculator"
+//   input_stream: "TENSORS:tensors"
+//   output_stream: "CLASSIFICATIONS:classifications"
+//   options: {
+//     [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
+//       num_classes: 1024
+//       min_score_threshold: 0.1
+//       label_map_path: "labelmap.txt"
+//     }
+//   }
+// }
+class TfLiteTensorsToClassificationCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions options_;
+  int top_k_ = 0;
+  std::unordered_map<int, std::string> label_map_;
+  bool label_map_loaded_ = false;
+};
+REGISTER_CALCULATOR(TfLiteTensorsToClassificationCalculator);
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag("TENSORS")) {
+    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  }
+
+  if (cc->Outputs().HasTag("CLASSIFICATIONS")) {
+    cc->Outputs().Tag("CLASSIFICATIONS").Set<ClassificationList>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Open(
+    CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  options_ = cc->Options<
+      ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions>();
+
+  top_k_ = options_.top_k();
+  if (options_.has_label_map_path()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path,
+                     PathToResourceAsFile(options_.label_map_path()));
+    std::string label_map_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+
+    std::istringstream stream(label_map_string);
+    std::string line;
+    int i = 0;
+    while (std::getline(stream, line)) {
+      label_map_[i++] = line;
+    }
+    label_map_loaded_ = true;
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Process(
+    CalculatorContext* cc) {
+  const auto& input_tensors =
+      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+
+  RET_CHECK_EQ(input_tensors.size(), 1);
+
+  const TfLiteTensor* raw_score_tensor = &input_tensors[0];
+  int num_classes = 1;
+  for (int i = 0; i < raw_score_tensor->dims->size; ++i) {
+    num_classes *= raw_score_tensor->dims->data[i];
+  }
+
+  if (label_map_loaded_) {
+    RET_CHECK_EQ(num_classes, label_map_.size());
+  }
+  const float* raw_scores = raw_score_tensor->data.f;
+
+  auto classification_list = absl::make_unique<ClassificationList>();
+  for (int i = 0; i < num_classes; ++i) {
+    if (options_.has_min_score_threshold() &&
+        raw_scores[i] < options_.min_score_threshold()) {
+      continue;
+    }
+    Classification* classification = classification_list->add_classification();
+    classification->set_index(i);
+    classification->set_score(raw_scores[i]);
+    if (label_map_loaded_) {
+      classification->set_label(label_map_[i]);
+    }
+  }
+
+  // Note that partial_sort will raise error when top_k_ >
+  // classification_list->classification_size().
+  CHECK_GE(classification_list->classification_size(), top_k_);
+  auto raw_classification_list = classification_list->mutable_classification();
+  if (top_k_ > 0 && classification_list->classification_size() >= top_k_) {
+    std::partial_sort(raw_classification_list->begin(),
+                      raw_classification_list->begin() + top_k_,
+                      raw_classification_list->end(),
+                      [](const Classification a, const Classification b) {
+                        return a.score() > b.score();
+                      });
+
+    // Resizes the underlying list to have only top_k_ classifications.
+    raw_classification_list->DeleteSubrange(
+        top_k_, raw_classification_list->size() - top_k_);
+  }
+  cc->Outputs()
+      .Tag("CLASSIFICATIONS")
+      .Add(classification_list.release(), cc->InputTimestamp());
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Close(
+    CalculatorContext* cc) {
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe