Project import generated by Copybara.

PiperOrigin-RevId: 267274408

mediapipe/calculators/audio/BUILD

@@ -68,6 +68,26 @@ mediapipe_cc_proto_library(
 )
 
 proto_library(
+<<<<<<< HEAD
+=======
+    name = "stabilized_log_calculator_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+mediapipe_cc_proto_library(
+    name = "stabilized_log_calculator_cc_proto",
+    srcs = ["stabilized_log_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":stabilized_log_calculator_proto"],
+)
+
+proto_library(
+>>>>>>> Project import generated by Copybara.
     name = "time_series_framer_calculator_proto",
     srcs = ["time_series_framer_calculator.proto"],
     visibility = ["//visibility:public"],
@@ -157,6 +177,25 @@ cc_library(
 )
 
 cc_library(
+<<<<<<< HEAD
+=======
+    name = "stabilized_log_calculator",
+    srcs = ["stabilized_log_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_util",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+>>>>>>> Project import generated by Copybara.
     name = "spectrogram_calculator",
     srcs = ["spectrogram_calculator.cc"],
     visibility = ["//visibility:public"],
@@ -267,6 +306,27 @@ cc_test(
 )
 
 cc_test(
+<<<<<<< HEAD
+=======
+    name = "stabilized_log_calculator_test",
+    srcs = ["stabilized_log_calculator_test.cc"],
+    deps = [
+        ":stabilized_log_calculator",
+        ":stabilized_log_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:time_series_test_util",
+        "@eigen_archive//:eigen",
+    ],
+)
+
+cc_test(
+>>>>>>> Project import generated by Copybara.
     name = "time_series_framer_calculator_test",
     srcs = ["time_series_framer_calculator_test.cc"],
     deps = [

mediapipe/calculators/audio/audio_decoder_calculator.cc

@@ -64,7 +64,11 @@ class AudioDecoderCalculator : public CalculatorBase {
 
   cc->Outputs().Tag("AUDIO").Set<Matrix>();
   if (cc->Outputs().HasTag("AUDIO_HEADER")) {
+<<<<<<< HEAD
     cc->Outputs().Tag("AUDIO_HEADER").Set<mediapipe::TimeSeriesHeader>();
+=======
+    cc->Outputs().Tag("AUDIO_HEADER").SetNone();
+>>>>>>> Project import generated by Copybara.
   }
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/audio/mfcc_mel_calculators.cc

@@ -90,8 +90,13 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
  private:
   // Takes header and options, and sets up state including calling
   // set_num_output_channels() on the base object.
+<<<<<<< HEAD
   virtual ::mediapipe::Status ConfigureTransform(
       const TimeSeriesHeader& header, const CalculatorOptions& options) = 0;
+=======
+  virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                                 CalculatorContext* cc) = 0;
+>>>>>>> Project import generated by Copybara.
 
   // Takes a vector<double> corresponding to an input frame, and
   // perform the specific transformation to produce an output frame.
@@ -108,7 +113,11 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
   RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
       cc->Inputs().Index(0).Header(), &input_header));
 
+<<<<<<< HEAD
   ::mediapipe::Status status = ConfigureTransform(input_header, cc->Options());
+=======
+  ::mediapipe::Status status = ConfigureTransform(input_header, cc);
+>>>>>>> Project import generated by Copybara.
 
   auto output_header = new TimeSeriesHeader(input_header);
   output_header->set_num_channels(num_output_channels_);
@@ -175,11 +184,17 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
   }
 
  private:
+<<<<<<< HEAD
   ::mediapipe::Status ConfigureTransform(
       const TimeSeriesHeader& header,
       const CalculatorOptions& options) override {
     MfccCalculatorOptions mfcc_options;
     time_series_util::FillOptionsExtensionOrDie(options, &mfcc_options);
+=======
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
+    MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
     mfcc_.reset(new audio_dsp::Mfcc());
     int input_length = header.num_channels();
     // Set up the parameters to the Mfcc object.
@@ -235,11 +250,18 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
   }
 
  private:
+<<<<<<< HEAD
   ::mediapipe::Status ConfigureTransform(
       const TimeSeriesHeader& header,
       const CalculatorOptions& options) override {
     MelSpectrumCalculatorOptions mel_spectrum_options;
     time_series_util::FillOptionsExtensionOrDie(options, &mel_spectrum_options);
+=======
+  ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
+                                         CalculatorContext* cc) override {
+    MelSpectrumCalculatorOptions mel_spectrum_options =
+        cc->Options<MelSpectrumCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
     mel_filterbank_.reset(new audio_dsp::MelFilterbank());
     int input_length = header.num_channels();
     set_num_output_channels(mel_spectrum_options.channel_count());

mediapipe/calculators/audio/rational_factor_resample_calculator.cc

@@ -64,8 +64,13 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
 
 ::mediapipe::Status RationalFactorResampleCalculator::Open(
     CalculatorContext* cc) {
+<<<<<<< HEAD
   RationalFactorResampleCalculatorOptions resample_options;
   time_series_util::FillOptionsExtensionOrDie(cc->Options(), &resample_options);
+=======
+  RationalFactorResampleCalculatorOptions resample_options =
+      cc->Options<RationalFactorResampleCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
 
   if (!resample_options.has_target_sample_rate()) {
     return tool::StatusInvalid(

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -71,10 +71,15 @@ class SpectrogramCalculator : public CalculatorBase {
         // Input stream with TimeSeriesHeader.
     );
 
+<<<<<<< HEAD
     SpectrogramCalculatorOptions spectrogram_options;
     time_series_util::FillOptionsExtensionOrDie(cc->Options(),
                                                 &spectrogram_options);
 
+=======
+    SpectrogramCalculatorOptions spectrogram_options =
+        cc->Options<SpectrogramCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
     if (!spectrogram_options.allow_multichannel_input()) {
       if (spectrogram_options.output_type() ==
           SpectrogramCalculatorOptions::COMPLEX) {
@@ -172,9 +177,14 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
 ::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
+<<<<<<< HEAD
   SpectrogramCalculatorOptions spectrogram_options;
   time_series_util::FillOptionsExtensionOrDie(cc->Options(),
                                               &spectrogram_options);
+=======
+  SpectrogramCalculatorOptions spectrogram_options =
+      cc->Options<SpectrogramCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
 
   if (spectrogram_options.frame_duration_seconds() <= 0.0) {
     ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
@@ -223,6 +233,13 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
 
   std::vector<double> window;
   switch (spectrogram_options.window_type()) {
+<<<<<<< HEAD
+=======
+    case SpectrogramCalculatorOptions::COSINE:
+      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                                                   &window);
+      break;
+>>>>>>> Project import generated by Copybara.
     case SpectrogramCalculatorOptions::HANN:
       audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
                                                  &window);

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -58,6 +58,10 @@ message SpectrogramCalculatorOptions {
   enum WindowType {
     HANN = 0;
     HAMMING = 1;
+<<<<<<< HEAD
+=======
+    COSINE = 2;
+>>>>>>> Project import generated by Copybara.
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/audio/stabilized_log_calculator.cc

@@ -0,0 +1,94 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines StabilizedLogCalculator.
+
+#include <cmath>
+#include <memory>
+#include <string>
+
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// Example config:
+// node {
+//   calculator: "StabilizedLogCalculator"
+//   input_stream: "input_time_series"
+//   output_stream: "stabilized_log_time_series"
+//   options {
+//     [mediapipe.StabilizedLogCalculatorOptions.ext] {
+//       stabilizer: .00001
+//       check_nonnegativity: true
+//     }
+//   }
+// }
+class StabilizedLogCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input stream with TimeSeriesHeader.
+    );
+    cc->Outputs().Index(0).Set<Matrix>(
+        // Output stabilized log stream with TimeSeriesHeader.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    StabilizedLogCalculatorOptions stabilized_log_calculator_options =
+        cc->Options<StabilizedLogCalculatorOptions>();
+
+    stabilizer_ = stabilized_log_calculator_options.stabilizer();
+    output_scale_ = stabilized_log_calculator_options.output_scale();
+    check_nonnegativity_ =
+        stabilized_log_calculator_options.check_nonnegativity();
+    CHECK_GE(stabilizer_, 0.0)
+        << "stabilizer must be >= 0.0, received a value of " << stabilizer_;
+
+    // If the input packets have a header, propagate the header to the output.
+    if (!cc->Inputs().Index(0).Header().IsEmpty()) {
+      TimeSeriesHeader input_header;
+      RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
+          cc->Inputs().Index(0).Header(), &input_header));
+      cc->Outputs().Index(0).SetHeader(
+          Adopt(new TimeSeriesHeader(input_header)));
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
+    if (check_nonnegativity_) {
+      CHECK_GE(input_matrix.minCoeff(), 0);
+    }
+    std::unique_ptr<Matrix> output_frame(new Matrix(
+        output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
+    cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  float stabilizer_;
+  bool check_nonnegativity_;
+  double output_scale_;
+};
+REGISTER_CALCULATOR(StabilizedLogCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/audio/stabilized_log_calculator.proto

@@ -0,0 +1,37 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message StabilizedLogCalculatorOptions {
+  extend CalculatorOptions {
+    optional StabilizedLogCalculatorOptions ext = 101978339;
+  }
+
+  // The calculator computes log(x + stabilizer). stabilizer must be >=
+  // 0, with 0 indicating a lack of stabilization.
+  optional float stabilizer = 1 [default = .00001];
+
+  // If true, CHECK that all input values in are >= 0. If false, the
+  // code will take the log of the potentially negative input values
+  // plus the stabilizer.
+  optional bool check_nonnegativity = 2 [default = true];
+
+  // Support a fixed multiplicative scaling of the output.
+  optional double output_scale = 3 [default = 1.0];
+}

mediapipe/calculators/audio/stabilized_log_calculator_test.cc

@@ -0,0 +1,131 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "Eigen/Core"
+#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+const float kStabilizer = 0.1;
+const int kNumChannels = 3;
+const int kNumSamples = 10;
+
+class StabilizedLogCalculatorTest
+    : public TimeSeriesCalculatorTest<StabilizedLogCalculatorOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "StabilizedLogCalculator";
+    options_.set_stabilizer(kStabilizer);
+
+    input_sample_rate_ = 8000.0;
+    num_input_channels_ = kNumChannels;
+    num_input_samples_ = kNumSamples;
+  }
+
+  void RunGraphNoReturn() { MEDIAPIPE_ASSERT_OK(RunGraph()); }
+};
+
+TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
+  const int kNumPackets = 5;
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        (input_data_matrices[output_packet].array() + kStabilizer).log(),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
+  const int kNumPackets = 5;
+  double output_scale = 2.5;
+  options_.set_output_scale(output_scale);
+
+  InitializeGraph();
+  FillInputHeader();
+
+  std::vector<Matrix> input_data_matrices;
+  for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
+    const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
+    Matrix input_data_matrix =
+        Matrix::Random(kNumChannels, kNumSamples).array().abs();
+    input_data_matrices.push_back(input_data_matrix);
+    AppendInputPacket(new Matrix(input_data_matrix), timestamp);
+  }
+
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
+    ExpectApproximatelyEqual(
+        output_scale *
+            ((input_data_matrices[output_packet].array() + kStabilizer).log()),
+        runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
+  }
+}
+
+TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)),
+                    0 /* timestamp */);
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  ExpectOutputHeaderEqualsInputHeader();
+  ExpectApproximatelyEqual(
+      Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(),
+      runner_->Outputs().Index(0).packets[0].Get<Matrix>());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesCheckFail) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  ASSERT_DEATH(RunGraphNoReturn(), "");
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {
+  options_.set_check_nonnegativity(false);
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
+      0 /* timestamp */);
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  // Results are undefined.
+}
+
+}  // namespace mediapipe

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -206,8 +206,13 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
 }
 
 ::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
+<<<<<<< HEAD
   TimeSeriesFramerCalculatorOptions framer_options;
   time_series_util::FillOptionsExtensionOrDie(cc->Options(), &framer_options);
+=======
+  TimeSeriesFramerCalculatorOptions framer_options =
+      cc->Options<TimeSeriesFramerCalculatorOptions>();
+>>>>>>> Project import generated by Copybara.
 
   RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0)
       << "Invalid or missing frame_duration_seconds. "

mediapipe/calculators/core/BUILD

@@ -162,6 +162,10 @@ cc_library(
     deps = [
         ":concatenate_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+<<<<<<< HEAD
+=======
+        "//mediapipe/framework/formats:landmark_cc_proto",
+>>>>>>> Project import generated by Copybara.
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@org_tensorflow//tensorflow/lite:framework",
@@ -523,6 +527,10 @@ cc_library(
     deps = [
         ":split_vector_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+<<<<<<< HEAD
+=======
+        "//mediapipe/framework/formats:landmark_cc_proto",
+>>>>>>> Project import generated by Copybara.
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",
@@ -628,6 +636,44 @@ cc_test(
 )
 
 cc_library(
+<<<<<<< HEAD
+=======
+    name = "matrix_to_vector_calculator",
+    srcs = ["matrix_to_vector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/tool:status_util",
+        "//mediapipe/util:time_series_util",
+        "@com_google_absl//absl/memory",
+        "@eigen_archive//:eigen",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "matrix_to_vector_calculator_test",
+    srcs = ["matrix_to_vector_calculator_test.cc"],
+    deps = [
+        ":matrix_to_vector_calculator",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+        "//mediapipe/util:time_series_test_util",
+        "//mediapipe/util:time_series_util",
+    ],
+)
+
+cc_library(
+>>>>>>> Project import generated by Copybara.
     name = "merge_calculator",
     srcs = ["merge_calculator.cc"],
     visibility = ["//visibility:public"],

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -16,6 +16,10 @@
 
 #include <vector>
 
+<<<<<<< HEAD
+=======
+#include "mediapipe/framework/formats/landmark.pb.h"
+>>>>>>> Project import generated by Copybara.
 #include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
@@ -41,4 +45,10 @@ typedef ConcatenateVectorCalculator<TfLiteTensor>
     ConcatenateTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
 
+<<<<<<< HEAD
+=======
+typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
+    ConcatenateLandmarkVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
+>>>>>>> Project import generated by Copybara.
 }  // namespace mediapipe

mediapipe/calculators/core/matrix_to_vector_calculator.cc

@@ -0,0 +1,83 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Defines MatrixToVectorCalculator.
+#include <math.h>
+
+#include <deque>
+#include <memory>
+#include <string>
+
+#include "Eigen/Core"
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+
+// A calculator that converts a Matrix M to a vector containing all the
+// entries of M in column-major order.
+//
+// Example config:
+// node {
+//   calculator: "MatrixToVectorCalculator"
+//   input_stream: "input_matrix"
+//   output_stream: "column_major_vector"
+// }
+class MatrixToVectorCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<Matrix>(
+        // Input Packet containing a Matrix.
+    );
+    cc->Outputs().Index(0).Set<std::vector<float>>(
+        // Output Packet containing a vector, one for each input Packet.
+    );
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+
+  // Outputs a packet containing a vector for each input packet.
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+};
+REGISTER_CALCULATOR(MatrixToVectorCalculator);
+
+::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
+  // Inform the framework that we don't alter timestamps.
+  cc->SetOffset(mediapipe::TimestampDiff(0));
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
+  const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
+  auto output = absl::make_unique<std::vector<float>>();
+
+  // The following lines work to convert the Matrix to a vector because Matrix
+  // is an Eigen::MatrixXf and Eigen uses column-major layout by default.
+  output->resize(input.rows() * input.cols());
+  auto output_as_matrix =
+      Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
+  output_as_matrix = input;
+
+  cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/matrix_to_vector_calculator_test.cc

@@ -0,0 +1,88 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "mediapipe/util/time_series_test_util.h"
+#include "mediapipe/util/time_series_util.h"
+
+namespace mediapipe {
+namespace {
+
+class MatrixToVectorCalculatorTest
+    : public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
+ protected:
+  void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
+
+  void AppendInput(const std::vector<float>& column_major_data,
+                   int64 timestamp) {
+    ASSERT_EQ(num_input_samples_ * num_input_channels_,
+              column_major_data.size());
+    Eigen::Map<const Matrix> data_map(&column_major_data[0],
+                                      num_input_channels_, num_input_samples_);
+    AppendInputPacket(new Matrix(data_map), timestamp);
+  }
+
+  void SetInputStreamParameters(int num_channels, int num_samples) {
+    num_input_channels_ = num_channels;
+    num_input_samples_ = num_samples;
+    input_sample_rate_ = 100;
+    input_packet_rate_ = 20.0;
+  }
+
+  void SetInputHeader(int num_channels, int num_samples) {
+    SetInputStreamParameters(num_channels, num_samples);
+    FillInputHeader();
+  }
+
+  void CheckOutputPacket(int packet, std::vector<float> expected_vector) {
+    const auto& actual_vector =
+        runner_->Outputs().Index(0).packets[packet].Get<std::vector<float>>();
+    EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector));
+  }
+};
+
+TEST_F(MatrixToVectorCalculatorTest, SingleRow) {
+  InitializeGraph();
+  SetInputHeader(1, 4);  // 1 channel x 4 samples
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0};
+  AppendInput(data_vector, 0);
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) {
+  InitializeGraph();
+  SetInputHeader(4, 2);  // 4 channels x 2 samples
+  // Actual data matrix is the transpose of the appearance below.
+  const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0,
+                                           5.0, 6.0, 7.0, 8.0};
+  AppendInput(data_vector, 0);
+
+  MEDIAPIPE_ASSERT_OK(RunGraph());
+  CheckOutputPacket(0, data_vector);
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -16,6 +16,10 @@
 
 #include <vector>
 
+<<<<<<< HEAD
+=======
+#include "mediapipe/framework/formats/landmark.pb.h"
+>>>>>>> Project import generated by Copybara.
 #include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
@@ -37,4 +41,10 @@ namespace mediapipe {
 typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
 REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
 
+<<<<<<< HEAD
+=======
+typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
+    SplitLandmarkVectorCalculator;
+REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
+>>>>>>> Project import generated by Copybara.
 }  // namespace mediapipe

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -244,7 +244,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     rotation_ = DegreesToRotationMode(
         cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
   } else {
+<<<<<<< HEAD
     rotation_ = DegreesToRotationMode(options_.rotation_mode());
+=======
+    rotation_ = options_.rotation_mode();
+>>>>>>> Project import generated by Copybara.
   }
 
   scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);

mediapipe/calculators/tensorflow/BUILD

@@ -188,6 +188,20 @@ mediapipe_cc_proto_library(
 )
 
 mediapipe_cc_proto_library(
+<<<<<<< HEAD
+=======
+    name = "tensorflow_session_from_frozen_graph_calculator_cc_proto",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+>>>>>>> Project import generated by Copybara.
     name = "tensorflow_session_from_saved_model_generator_cc_proto",
     srcs = ["tensorflow_session_from_saved_model_generator.proto"],
     cc_deps = ["//mediapipe/framework:packet_generator_cc_proto"],
@@ -445,6 +459,38 @@ cc_library(
 )
 
 cc_library(
+<<<<<<< HEAD
+=======
+    name = "tensorflow_session_from_frozen_graph_calculator",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator.cc"],
+    features = ["no_layering_check"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tensorflow_session",
+        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/tool:status_util",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:ret_check",
+    ] + select({
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+            "@org_tensorflow//tensorflow/core:core",
+        ],
+        "//mediapipe:android": [
+            "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
+            "//mediapipe/android/file/base",
+        ],
+        "//mediapipe:ios": [
+            "@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
+            "//mediapipe/android/file/base",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_library(
+>>>>>>> Project import generated by Copybara.
     name = "tensorflow_session_from_frozen_graph_generator",
     srcs = ["tensorflow_session_from_frozen_graph_generator.cc"],
     features = ["no_layering_check"],
@@ -738,6 +784,39 @@ cc_test(
 )
 
 cc_test(
+<<<<<<< HEAD
+=======
+    name = "tensorflow_session_from_frozen_graph_calculator_test",
+    srcs = ["tensorflow_session_from_frozen_graph_calculator_test.cc"],
+    data = [":test_frozen_graph"],
+    linkstatic = 1,
+    deps = [
+        ":tensorflow_inference_calculator",
+        ":tensorflow_session",
+        ":tensorflow_session_from_frozen_graph_calculator",
+        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/deps:file_path",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:tag_map_helper",
+        "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/strings",
+        "@org_tensorflow//tensorflow/core:direct_session",
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:testlib",
+        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
+        "@org_tensorflow//tensorflow/core/kernels:math",
+    ],
+)
+
+cc_test(
+>>>>>>> Project import generated by Copybara.
     name = "tensorflow_session_from_frozen_graph_generator_test",
     srcs = ["tensorflow_session_from_frozen_graph_generator_test.cc"],
     data = [":test_frozen_graph"],

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -34,6 +34,13 @@
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_util.h"
 
+<<<<<<< HEAD
+=======
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+#include "tensorflow/core/profiler/lib/traceme.h"
+#endif
+
+>>>>>>> Project import generated by Copybara.
 namespace tf = ::tensorflow;
 
 namespace mediapipe {
@@ -435,9 +442,21 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
       session_run_throttle->Acquire(1);
     }
     const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
+<<<<<<< HEAD
     const tf::Status tf_status =
         session_->Run(input_tensors, output_tensor_names,
                       {} /* target_node_names */, &outputs);
+=======
+    tf::Status tf_status;
+    {
+#if !defined(__ANDROID__) && !defined(__APPLE__)
+      tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
+#endif
+      tf_status = session_->Run(input_tensors, output_tensor_names,
+                                {} /* target_node_names */, &outputs);
+    }
+
+>>>>>>> Project import generated by Copybara.
     if (session_run_throttle != nullptr) {
       session_run_throttle->Release(1);
     }

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc

@@ -0,0 +1,136 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Reads serialized GraphDef proto. There are three ways to load a model:
+// 1. Specify the path to a graph.pb in the calculator options.
+// 2. Specify the path to the graph.pb through the
+// input_side_packet:STRING_MODEL_FILE_PATH
+// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL,
+// typically provided by EmbeddingFilePacketFactory.
+//
+// Produces a SessionBundle that TensorFlowInferenceCalculator can use.
+
+#include <string>
+
+#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/tool/status_util.h"
+#include "tensorflow/core/public/session_options.h"
+
+#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
+    defined(__APPLE__) && !TARGET_OS_OSX
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+namespace tf = ::tensorflow;
+
+class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    const auto& options =
+        cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
+    bool has_exactly_one_model =
+        !options.graph_proto_path().empty()
+            ? !(cc->InputSidePackets().HasTag("STRING_MODEL") |
+                cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"))
+            : (cc->InputSidePackets().HasTag("STRING_MODEL") ^
+               cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"));
+    RET_CHECK(has_exactly_one_model)
+        << "Must have exactly one of graph_proto_path in options or "
+           "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
+    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+      cc->InputSidePackets()
+          .Tag("STRING_MODEL")
+          .Set<std::string>(
+              // String model from embedded path
+          );
+    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+      cc->InputSidePackets()
+          .Tag("STRING_MODEL_FILE_PATH")
+          .Set<std::string>(
+              // Filename of std::string model.
+          );
+    }
+    cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>(
+        // A TensorFlow model loaded and ready for use along with
+        // a map from tags to tensor names.
+    );
+    RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    const auto& options =
+        cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
+    // Output bundle packet.
+    auto session = ::absl::make_unique<TensorFlowSession>();
+
+    tf::SessionOptions session_options;
+    session_options.config.CopyFrom(options.config());
+    std::vector<mediapipe::ProtoString> initialization_op_names;
+    initialization_op_names.reserve(options.initialization_op_names_size());
+    for (int i = 0; i < options.initialization_op_names_size(); ++i) {
+      initialization_op_names.emplace_back(options.initialization_op_names(i));
+    }
+    session->session.reset(tf::NewSession(session_options));
+
+    std::string graph_def_serialized;
+    if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
+      graph_def_serialized =
+          cc->InputSidePackets().Tag("STRING_MODEL").Get<std::string>();
+    } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
+      const std::string& frozen_graph = cc->InputSidePackets()
+                                            .Tag("STRING_MODEL_FILE_PATH")
+                                            .Get<std::string>();
+      RET_CHECK_OK(
+          mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
+    } else {
+      RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(),
+                                                &graph_def_serialized));
+    }
+    tensorflow::GraphDef graph_def;
+
+    RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
+    const tf::Status tf_status = session->session->Create(graph_def);
+    RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
+
+    for (const auto& key_value : options.tag_to_tensor_names()) {
+      session->tag_to_tensor_map[key_value.first] = key_value.second;
+    }
+    if (!initialization_op_names.empty()) {
+      const tf::Status tf_status =
+          session->session->Run({}, {}, initialization_op_names, {});
+      // RET_CHECK on the tf::Status object itself in order to print an
+      // informative error message.
+      RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
+    }
+
+    cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return ::mediapipe::OkStatus();
+  }
+};
+REGISTER_CALCULATOR(TensorFlowSessionFromFrozenGraphCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto

@@ -0,0 +1,72 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "tensorflow/core/protobuf/config.proto";
+
+message TensorFlowSessionFromFrozenGraphCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional TensorFlowSessionFromFrozenGraphCalculatorOptions ext = 266997877;
+  }
+
+  // Path to file containing serialized proto of type tensorflow::GraphDef.
+  optional string graph_proto_path = 1;
+
+  // To run inference with MediaPipe inputs MediaPipe streams need to be mapped
+  // to TensorFlow tensors. This map defines the which streams are fed into
+  // which tensors in the model. The MediaPipe tag of the stream is the map key.
+  // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING"
+  // and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of
+  // input_stream or output_stream of the TensorflowInferenceCalculator
+  // consuming the packet produced by this calculator. The tensor names must
+  // match the tensor names in the graph that you want to feed or fetch into or
+  // out of. Examples:  "DecodeJpeg/contents:0" or "softmax:0". For example, a
+  // mediapipe graph can include the nodes:
+  //
+  // node {
+  //   calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+  //   output_side_packet: "SESSION:session"
+  //   options {
+  //     [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+  //       graph_proto_path: "[PATH]"
+  //       tag_to_tensor_names {
+  //         key: "JPG_STRING"
+  //         value: "input:0"
+  //       }
+  //       tag_to_tensor_names {
+  //         key: "SOFTMAX"
+  //         value: "softmax:0"
+  //       }
+  //     }
+  //   }
+  // }
+  // node {
+  //   calculator: "TensorflowInferenceCalculator"
+  //   input_side_packet: "SESSION:graph_with_bindings"
+  //   input_stream: "JPG_STRING:jpg_string_tensor"
+  //   output_stream: "SOFTMAX:softmax_tensor"
+  // }
+  map<string, string> tag_to_tensor_names = 2;
+
+  // Tensorflow session config options.
+  optional tensorflow.ConfigProto config = 3;
+
+  // Graph nodes to run to initialize the model. Any output of these ops is
+  // ignored.
+  repeated string initialization_op_names = 4;
+}

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc

@@ -0,0 +1,316 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
+#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/tag_map_helper.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+namespace tf = ::tensorflow;
+
+std::string GetGraphDefPath() {
+  return mediapipe::file::JoinPath("./",
+                                   "mediapipe/calculators/tensorflow/"
+                                   "testdata/frozen_graph_def.pb");
+}
+
+// Helper function that creates Tensor INT32 matrix with size 1x3.
+tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) {
+  tf::Tensor tensor(tf::DT_INT32,
+                    tf::TensorShape(std::vector<tf::int64>({1, 3})));
+  auto matrix = tensor.matrix<int32>();
+  matrix(0, 0) = v1;
+  matrix(0, 1) = v2;
+  matrix(0, 2) = v3;
+  return tensor;
+}
+
+class TensorFlowSessionFromFrozenGraphCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    extendable_options_.Clear();
+    calculator_options_ = extendable_options_.MutableExtension(
+        TensorFlowSessionFromFrozenGraphCalculatorOptions::ext);
+    calculator_options_->set_graph_proto_path(GetGraphDefPath());
+    (*calculator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] =
+        "multiplied:0";
+    (*calculator_options_->mutable_tag_to_tensor_names())["A"] = "a:0";
+    (*calculator_options_->mutable_tag_to_tensor_names())["B"] = "b:0";
+    calculator_options_->mutable_config()->set_intra_op_parallelism_threads(1);
+    calculator_options_->mutable_config()->set_inter_op_parallelism_threads(2);
+  }
+
+  void VerifySignatureMap(const TensorFlowSession& session) {
+    // Session must be set.
+    ASSERT_NE(session.session, nullptr);
+
+    // Bindings are inserted.
+    EXPECT_EQ(session.tag_to_tensor_map.size(), 3);
+
+    // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators.
+    EXPECT_FALSE(session.tag_to_tensor_map.find("A") ==
+                 session.tag_to_tensor_map.end());
+    EXPECT_FALSE(session.tag_to_tensor_map.find("B") ==
+                 session.tag_to_tensor_map.end());
+    EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") ==
+                 session.tag_to_tensor_map.end());
+    // Sanity: find() actually returns a reference to end() if element not
+    // found.
+    EXPECT_TRUE(session.tag_to_tensor_map.find("Z") ==
+                session.tag_to_tensor_map.end());
+
+    EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0");
+    EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0");
+    EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0");
+  }
+
+  CalculatorOptions extendable_options_;
+  TensorFlowSessionFromFrozenGraphCalculatorOptions* calculator_options_;
+};
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CreatesPacketWithGraphAndBindings) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:tf_model"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+// Integration test. Verifies that TensorFlowInferenceCalculator correctly
+// consumes the Packet emitted by this calculator.
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       ProducesPacketUsableByTensorFlowInferenceCalculator) {
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          absl::Substitute(R"(
+      node {
+        calculator: "TensorFlowInferenceCalculator"
+        input_side_packet: "SESSION:session"
+        input_stream: "A:a_tensor"
+        output_stream: "MULTIPLIED:multiplied_tensor"
+        options {
+          [mediapipe.TensorFlowInferenceCalculatorOptions.ext] {
+            batch_size: 5
+            add_batch_dim_to_tensors: false
+          }
+        }
+      }
+
+      node {
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        }
+      }
+      input_stream: "a_tensor"
+  )",
+                           calculator_options_->DebugString()));
+
+  CalculatorGraph graph;
+  MEDIAPIPE_ASSERT_OK(graph.Initialize(config));
+  StatusOrPoller status_or_poller =
+      graph.AddOutputStreamPoller("multiplied_tensor");
+  ASSERT_TRUE(status_or_poller.ok());
+  OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
+
+  MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
+  MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
+      "a_tensor",
+      Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
+  MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor"));
+
+  Packet packet;
+  ASSERT_TRUE(poller.Next(&packet));
+  // input tensor gets multiplied by [[3, 2, 1]]. Expected output:
+  tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10);
+  EXPECT_EQ(expected_multiplication.DebugString(),
+            packet.Get<tf::Tensor>().DebugString());
+
+  ASSERT_FALSE(poller.Next(&packet));
+  MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CreatesPacketWithGraphAndBindingsFromInputSidePacket) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+
+  std::string serialized_graph_contents;
+  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                                   &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+TEST_F(
+    TensorFlowSessionFromFrozenGraphCalculatorTest,
+    CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForOptionsAndInputsProvideGraphDefProto) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForAllInputsProvideGraphDefProto) {
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  std::string serialized_graph_contents;
+  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                                   &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) {
+  calculator_options_->clear_graph_proto_path();
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
+        input_side_packet: "STRING_MODEL:model"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
+      Adopt(new std::string(GetGraphDefPath()));
+  std::string serialized_graph_contents;
+  MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
+                                                   &serialized_graph_contents));
+  runner.MutableSidePackets()->Tag("STRING_MODEL") =
+      Adopt(new std::string(serialized_graph_contents));
+  auto run_status = runner.Run();
+  EXPECT_THAT(
+      run_status.message(),
+      ::testing::HasSubstr("Must have exactly one of graph_proto_path"));
+}
+
+TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
+       CheckInitializationOpName) {
+  calculator_options_->add_initialization_op_names("multiplied:0");
+  CalculatorRunner runner(absl::Substitute(R"(
+        calculator: "TensorFlowSessionFromFrozenGraphCalculator"
+        output_side_packet: "SESSION:session"
+        options {
+          [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
+            $0
+          }
+        })",
+                                           calculator_options_->DebugString()));
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const TensorFlowSession& session =
+      runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
+  VerifySignatureMap(session);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tflite/BUILD

@@ -62,6 +62,16 @@ proto_library(
 )
 
 proto_library(
+<<<<<<< HEAD
+=======
+    name = "tflite_tensors_to_classification_calculator_proto",
+    srcs = ["tflite_tensors_to_classification_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+>>>>>>> Project import generated by Copybara.
     name = "tflite_tensors_to_landmarks_calculator_proto",
     srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
     visibility = ["//visibility:public"],
@@ -117,6 +127,17 @@ mediapipe_cc_proto_library(
 )
 
 mediapipe_cc_proto_library(
+<<<<<<< HEAD
+=======
+    name = "tflite_tensors_to_classification_calculator_cc_proto",
+    srcs = ["tflite_tensors_to_classification_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [":tflite_tensors_to_classification_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+>>>>>>> Project import generated by Copybara.
     name = "tflite_tensors_to_landmarks_calculator_cc_proto",
     srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
@@ -311,6 +332,28 @@ cc_library(
     alwayslink = 1,
 )
 
+<<<<<<< HEAD
+=======
+cc_test(
+    name = "tflite_tensors_to_classification_calculator_test",
+    srcs = ["tflite_tensors_to_classification_calculator_test.cc"],
+    data = ["testdata/labelmap.txt"],
+    deps = [
+        ":tflite_tensors_to_classification_calculator",
+        ":tflite_tensors_to_classification_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_googletest//:gtest_main",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+)
+
+>>>>>>> Project import generated by Copybara.
 cc_library(
     name = "tflite_tensors_to_detections_calculator",
     srcs = ["tflite_tensors_to_detections_calculator.cc"],
@@ -340,6 +383,40 @@ cc_library(
 )
 
 cc_library(
+<<<<<<< HEAD
+=======
+    name = "tflite_tensors_to_classification_calculator",
+    srcs = ["tflite_tensors_to_classification_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tflite_tensors_to_classification_calculator_cc_proto",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
+        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:location",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util:resource_util",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ] + select({
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:apple": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+    }),
+    alwayslink = 1,
+)
+
+cc_library(
+>>>>>>> Project import generated by Copybara.
     name = "tflite_tensors_to_landmarks_calculator",
     srcs = ["tflite_tensors_to_landmarks_calculator.cc"],
     visibility = ["//visibility:public"],

mediapipe/calculators/tflite/testdata/labelmap.txt

@@ -0,0 +1,3 @@
+classA
+classB
+classC

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -434,8 +434,14 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     use_quantized_tensors_ = false;
   } else {
     RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
+<<<<<<< HEAD
     use_quantized_tensors_ = (interpreter_->tensor(0)->quantization.type ==
                               kTfLiteAffineQuantization);
+=======
+    use_quantized_tensors_ =
+        (interpreter_->tensor(interpreter_->inputs()[0])->quantization.type ==
+         kTfLiteAffineQuantization);
+>>>>>>> Project import generated by Copybara.
     if (use_quantized_tensors_) gpu_inference_ = false;
   }
 

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc

@@ -0,0 +1,176 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <unordered_map>
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "absl/types/span.h"
+#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/resource_util.h"
+#include "tensorflow/lite/interpreter.h"
+#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
+#include "mediapipe/util/android/file/base/file.h"
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+// Convert result TFLite tensors from classification models into MediaPipe
+// classifications.
+//
+// Input:
+//  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 containing one
+//            tensor, the size of which must be (1, * num_classes).
+// Output:
+//  CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index
+//                    fields of each classification are set, while the label
+//                    field is only set if label_map_path is provided.
+//
+// Usage example:
+// node {
+//   calculator: "TfLiteTensorsToClassificationCalculator"
+//   input_stream: "TENSORS:tensors"
+//   output_stream: "CLASSIFICATIONS:classifications"
+//   options: {
+//     [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
+//       num_classes: 1024
+//       min_score_threshold: 0.1
+//       label_map_path: "labelmap.txt"
+//     }
+//   }
+// }
+class TfLiteTensorsToClassificationCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  int top_k_ = 0;
+  double min_score_threshold_ = 0;
+  std::unordered_map<int, std::string> label_map_;
+  bool label_map_loaded_ = false;
+};
+REGISTER_CALCULATOR(TfLiteTensorsToClassificationCalculator);
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::GetContract(
+    CalculatorContract* cc) {
+  RET_CHECK(!cc->Inputs().GetTags().empty());
+  RET_CHECK(!cc->Outputs().GetTags().empty());
+
+  if (cc->Inputs().HasTag("TENSORS")) {
+    cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
+  }
+
+  if (cc->Outputs().HasTag("CLASSIFICATIONS")) {
+    cc->Outputs().Tag("CLASSIFICATIONS").Set<ClassificationList>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Open(
+    CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
+
+  auto options = cc->Options<
+      ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions>();
+
+  top_k_ = options.top_k();
+  min_score_threshold_ = options.min_score_threshold();
+  if (options.has_label_map_path()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path,
+                     PathToResourceAsFile(options.label_map_path()));
+    std::string label_map_string;
+    RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
+
+    std::istringstream stream(label_map_string);
+    std::string line;
+    int i = 0;
+    while (std::getline(stream, line)) {
+      label_map_[i++] = line;
+    }
+    label_map_loaded_ = true;
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Process(
+    CalculatorContext* cc) {
+  const auto& input_tensors =
+      cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
+
+  RET_CHECK_EQ(input_tensors.size(), 1);
+
+  const TfLiteTensor* raw_score_tensor = &input_tensors[0];
+  RET_CHECK_EQ(raw_score_tensor->dims->size, 2);
+  RET_CHECK_EQ(raw_score_tensor->dims->data[0], 1);
+  int num_classes = raw_score_tensor->dims->data[1];
+  if (label_map_loaded_) {
+    RET_CHECK_EQ(num_classes, label_map_.size());
+  }
+  const float* raw_scores = raw_score_tensor->data.f;
+
+  auto classification_list = absl::make_unique<ClassificationList>();
+  for (int i = 0; i < num_classes; ++i) {
+    if (raw_scores[i] < min_score_threshold_) {
+      continue;
+    }
+    Classification* classification = classification_list->add_classification();
+    classification->set_index(i);
+    classification->set_score(raw_scores[i]);
+    if (label_map_loaded_) {
+      classification->set_label(label_map_[i]);
+    }
+  }
+
+  // Note that partial_sort will raise error when top_k_ >
+  // classification_list->classification_size().
+  auto raw_classification_list = classification_list->mutable_classification();
+  if (top_k_ > 0 && classification_list->classification_size() >= top_k_) {
+    std::partial_sort(raw_classification_list->begin(),
+                      raw_classification_list->begin() + top_k_,
+                      raw_classification_list->end(),
+                      [](const Classification a, const Classification b) {
+                        return a.score() > b.score();
+                      });
+
+    // Resizes the underlying list to have only top_k_ classifications.
+    raw_classification_list->DeleteSubrange(
+        top_k_, raw_classification_list->size() - top_k_);
+  }
+  cc->Outputs()
+      .Tag("CLASSIFICATIONS")
+      .Add(classification_list.release(), cc->InputTimestamp());
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status TfLiteTensorsToClassificationCalculator::Close(
+    CalculatorContext* cc) {
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto

@@ -0,0 +1,35 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The option proto for the TfLiteTensorsToClassificationCalculator.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message TfLiteTensorsToClassificationCalculatorOptions {
+  extend .mediapipe.CalculatorOptions {
+    optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
+  }
+
+  // Score threshold for perserving the class.
+  optional float min_score_threshold = 1;
+  // Number of highest scoring labels to output.  If top_k is not positive then
+  // all labels are used.
+  optional int32 top_k = 2;
+  // Path to a label map file for getting the actual name of class ids.
+  optional string label_map_path = 3;
+}

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc

@@ -0,0 +1,194 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+using ::mediapipe::ParseTextProtoOrDie;
+using ::tflite::Interpreter;
+using Node = ::mediapipe::CalculatorGraphConfig::Node;
+
+class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test {
+ protected:
+  void BuildGraph(mediapipe::CalculatorRunner* runner,
+                  const std::vector<float>& scores) {
+    interpreter_ = absl::make_unique<Interpreter>();
+
+    std::vector<int> dims(2);
+    dims[0] = 1;
+    dims[1] = scores.size();
+
+    interpreter_->AddTensors(1);
+    interpreter_->SetInputs({0});
+    interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", dims,
+                                               TfLiteQuantization());
+
+    int t = interpreter_->inputs()[0];
+    TfLiteTensor* tensor = interpreter_->tensor(t);
+    interpreter_->ResizeInputTensor(t, dims);
+    interpreter_->AllocateTensors();
+
+    float* tensor_buffer = tensor->data.f;
+    ASSERT_NE(tensor_buffer, nullptr);
+    for (int i = 0; i < scores.size(); ++i) {
+      tensor_buffer[i] = scores[i];
+    }
+
+    auto tensors = absl::make_unique<std::vector<TfLiteTensor>>();
+    tensors->emplace_back(*tensor);
+
+    int64 stream_timestamp = 0;
+    auto& input_stream_packets =
+        runner->MutableInputs()->Tag("TENSORS").packets;
+
+    input_stream_packets.push_back(
+        mediapipe::Adopt(tensors.release())
+            .At(mediapipe::Timestamp(stream_timestamp++)));
+  }
+
+  std::unique_ptr<Interpreter> interpreter_;
+};
+
+TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+    calculator: "TfLiteTensorsToClassificationCalculator"
+    input_stream: "TENSORS:tensors"
+    output_stream: "CLASSIFICATIONS:classifications"
+    options {
+      [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {}
+    }
+  )"));
+
+  BuildGraph(&runner, {0, 0.5, 1});
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
+
+  EXPECT_EQ(1, output_packets_.size());
+
+  const auto& classification_list =
+      output_packets_[0].Get<ClassificationList>();
+  EXPECT_EQ(3, classification_list.classification_size());
+
+  // Verify that the label_id and score fields are set correctly.
+  for (int i = 0; i < classification_list.classification_size(); ++i) {
+    EXPECT_EQ(i, classification_list.classification(i).index());
+    EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
+    ASSERT_FALSE(classification_list.classification(i).has_label());
+  }
+}
+
+TEST_F(TfLiteTensorsToClassificationCalculatorTest,
+       CorrectOutputWithLabelMapPath) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+    calculator: "TfLiteTensorsToClassificationCalculator"
+    input_stream: "TENSORS:tensors"
+    output_stream: "CLASSIFICATIONS:classifications"
+    options {
+      [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
+        label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt"
+      }
+    }
+  )"));
+
+  BuildGraph(&runner, {0, 0.5, 1});
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
+
+  EXPECT_EQ(1, output_packets_.size());
+
+  const auto& classification_list =
+      output_packets_[0].Get<ClassificationList>();
+  EXPECT_EQ(3, classification_list.classification_size());
+
+  // Verify that the label field is set.
+  for (int i = 0; i < classification_list.classification_size(); ++i) {
+    EXPECT_EQ(i, classification_list.classification(i).index());
+    EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
+    ASSERT_TRUE(classification_list.classification(i).has_label());
+  }
+}
+
+TEST_F(TfLiteTensorsToClassificationCalculatorTest,
+       CorrectOutputWithLabelMinScoreThreshold) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+    calculator: "TfLiteTensorsToClassificationCalculator"
+    input_stream: "TENSORS:tensors"
+    output_stream: "CLASSIFICATIONS:classifications"
+    options {
+      [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
+        min_score_threshold: 0.6
+      }
+    }
+  )"));
+
+  BuildGraph(&runner, {0, 0.5, 1});
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
+
+  EXPECT_EQ(1, output_packets_.size());
+
+  const auto& classification_list =
+      output_packets_[0].Get<ClassificationList>();
+
+  // Verify that the low score labels are filtered out.
+  EXPECT_EQ(1, classification_list.classification_size());
+  EXPECT_EQ(1, classification_list.classification(0).score());
+}
+
+TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
+    calculator: "TfLiteTensorsToClassificationCalculator"
+    input_stream: "TENSORS:tensors"
+    output_stream: "CLASSIFICATIONS:classifications"
+    options {
+      [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
+        top_k: 2
+      }
+    }
+  )"));
+
+  BuildGraph(&runner, {0, 0.5, 1});
+  MEDIAPIPE_ASSERT_OK(runner.Run());
+
+  const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
+
+  EXPECT_EQ(1, output_packets_.size());
+
+  const auto& classification_list =
+      output_packets_[0].Get<ClassificationList>();
+
+  // Verify that the only top2 labels are left.
+  EXPECT_EQ(2, classification_list.classification_size());
+  for (int i = 0; i < classification_list.classification_size(); ++i) {
+    EXPECT_EQ((classification_list.classification_size() - i) * 0.5,
+              classification_list.classification(i).score());
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/opencv_video_decoder_calculator.cc

@@ -154,8 +154,19 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
                      cv::COLOR_BGRA2RGBA);
       }
     }
+<<<<<<< HEAD
     cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp);
     decoded_frames_++;
+=======
+    // If the timestamp of the current frame is not greater than the one of the
+    // previous frame, the new frame will be discarded.
+    if (prev_timestamp_ < timestamp) {
+      cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp);
+      prev_timestamp_ = timestamp;
+      decoded_frames_++;
+    }
+
+>>>>>>> Project import generated by Copybara.
     return ::mediapipe::OkStatus();
   }
 
@@ -178,6 +189,10 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
   int frame_count_;
   int decoded_frames_ = 0;
   ImageFormat::Format format_;
+<<<<<<< HEAD
+=======
+  Timestamp prev_timestamp_ = Timestamp::Unset();
+>>>>>>> Project import generated by Copybara.
 };
 
 REGISTER_CALCULATOR(OpenCvVideoDecoderCalculator);