Project import generated by Copybara.

GitOrigin-RevId: 1e13be30e2c6838d4a2ff768a39c414bc80534bb

mediapipe/calculators/core/BUILD

@@ -151,6 +151,16 @@ mediapipe_proto_library(
     ],
 )
 
+mediapipe_proto_library(
+    name = "get_vector_item_calculator_proto",
+    srcs = ["get_vector_item_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 cc_library(
     name = "add_header_calculator",
     srcs = ["add_header_calculator.cc"],
@@ -561,6 +571,7 @@ cc_test(
     name = "packet_cloner_calculator_test",
     srcs = ["packet_cloner_calculator_test.cc"],
     deps = [
+        ":gate_calculator",
         ":packet_cloner_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
@@ -1281,6 +1292,7 @@ cc_library(
     hdrs = ["get_vector_item_calculator.h"],
     visibility = ["//visibility:public"],
     deps = [
+        ":get_vector_item_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/api2:node",
@@ -1293,6 +1305,20 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "get_vector_item_calculator_test",
+    srcs = ["get_vector_item_calculator_test.cc"],
+    deps = [
+        ":get_vector_item_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/strings:str_format",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_library(
     name = "vector_size_calculator",
     srcs = ["vector_size_calculator.cc"],
@@ -1307,3 +1333,54 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "packet_sequencer_calculator",
+    srcs = ["packet_sequencer_calculator.cc"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:contract",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "packet_sequencer_calculator_test",
+    srcs = ["packet_sequencer_calculator_test.cc"],
+    deps = [
+        ":packet_sequencer_calculator",
+        "//mediapipe/calculators/core:pass_through_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:subgraph",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_library(
+    name = "merge_to_vector_calculator",
+    srcs = ["merge_to_vector_calculator.cc"],
+    hdrs = ["merge_to_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:image",
+        "@com_google_absl//absl/status",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/core/clip_vector_size_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message ClipVectorSizeCalculatorOptions {
   extend CalculatorOptions {
     optional ClipVectorSizeCalculatorOptions ext = 274674998;

mediapipe/calculators/core/concatenate_vector_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message ConcatenateVectorCalculatorOptions {
   extend CalculatorOptions {
     optional ConcatenateVectorCalculatorOptions ext = 259397839;

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -20,8 +20,6 @@ import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
 import "mediapipe/framework/formats/landmark.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message ConstantSidePacketCalculatorOptions {
   extend CalculatorOptions {
     optional ConstantSidePacketCalculatorOptions ext = 291214597;

mediapipe/calculators/core/dequantize_byte_array_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message DequantizeByteArrayCalculatorOptions {
   extend CalculatorOptions {
     optional DequantizeByteArrayCalculatorOptions ext = 272316343;

mediapipe/calculators/core/flow_limiter_calculator.proto

@@ -18,7 +18,8 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
+option java_package = "com.google.mediapipe.calculator.proto";
+option java_outer_classname = "FlowLimiterCalculatorProto";
 
 message FlowLimiterCalculatorOptions {
   extend mediapipe.CalculatorOptions {

mediapipe/calculators/core/gate_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message GateCalculatorOptions {
   extend mediapipe.CalculatorOptions {
     optional GateCalculatorOptions ext = 261754847;

mediapipe/calculators/core/get_vector_item_calculator.h

@@ -17,22 +17,24 @@
 
 #include <optional>
 
+#include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
 namespace api2 {
 
-// A calcutlator to return an item from the vector by its index.
+// A calculator to return an item from the vector by its index.
+// Item index can be specified through INDEX stream and/or calculator options.
+// INDEX stream takes precedence over options.
 //
 // Inputs:
 //   VECTOR - std::vector<T>
 //     Vector to take an item from.
-//   INDEX - int
+//   INDEX [OPTIONAL] - int
 //     Index of the item to return.
 //
 // Outputs:
@@ -45,26 +47,47 @@ namespace api2 {
 //     input_stream: "VECTOR:vector"
 //     input_stream: "INDEX:index"
 //     input_stream: "ITEM:item"
+//     options {
+//       [mediapipe.GetVectorItemCalculatorOptions.ext] {
+//         item_index: 5
+//       }
+//     }
 //   }
 //
 template <typename T>
 class GetVectorItemCalculator : public Node {
  public:
   static constexpr Input<std::vector<T>> kIn{"VECTOR"};
-  static constexpr Input<int> kIdx{"INDEX"};
+  static constexpr Input<int>::Optional kIdx{"INDEX"};
   static constexpr Output<T> kOut{"ITEM"};
 
   MEDIAPIPE_NODE_CONTRACT(kIn, kIdx, kOut);
 
+  absl::Status Open(CalculatorContext* cc) final {
+    auto& options = cc->Options<mediapipe::GetVectorItemCalculatorOptions>();
+    RET_CHECK(kIdx(cc).IsConnected() || options.has_item_index());
+    return absl::OkStatus();
+  }
+
   absl::Status Process(CalculatorContext* cc) final {
-    if (kIn(cc).IsEmpty() || kIdx(cc).IsEmpty()) {
+    if (kIn(cc).IsEmpty()) {
       return absl::OkStatus();
     }
 
     const std::vector<T>& items = kIn(cc).Get();
-    const int idx = kIdx(cc).Get();
+    const auto& options =
+        cc->Options<mediapipe::GetVectorItemCalculatorOptions>();
 
-    RET_CHECK_LT(idx, items.size());
+    int idx = 0;
+    if (kIdx(cc).IsConnected() && !kIdx(cc).IsEmpty()) {
+      idx = kIdx(cc).Get();
+    } else if (options.has_item_index()) {
+      idx = options.item_index();
+    } else {
+      return absl::OkStatus();
+    }
+
+    RET_CHECK(idx >= 0 && idx < items.size());
     kOut(cc).Send(items[idx]);
 
     return absl::OkStatus();

mediapipe/calculators/core/get_vector_item_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message GetVectorItemCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional GetVectorItemCalculatorOptions ext = 463538543;
+  }
+
+  // Index of vector item to get. INDEX input stream can be used instead, or to
+  // override.
+  optional int32 item_index = 1;
+}

mediapipe/calculators/core/get_vector_item_calculator_test.cc

@@ -0,0 +1,230 @@
+#include "mediapipe/calculators/core/get_vector_item_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_format.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+MATCHER_P(IntPacket, value, "") {
+  return testing::Value(arg.template Get<int>(), testing::Eq(value));
+}
+
+MATCHER_P(TimestampValue, value, "") {
+  return testing::Value(arg.Timestamp(), testing::Eq(Timestamp(value)));
+}
+
+using TestGetIntVectorItemCalculator = api2::GetVectorItemCalculator<int>;
+MEDIAPIPE_REGISTER_NODE(TestGetIntVectorItemCalculator);
+
+CalculatorRunner MakeRunnerWithStream() {
+  return CalculatorRunner(R"(
+    calculator: "TestGetIntVectorItemCalculator"
+    input_stream: "VECTOR:vector_stream"
+    input_stream: "INDEX:index_stream"
+    output_stream: "ITEM:item_stream"
+  )");
+}
+
+CalculatorRunner MakeRunnerWithOptions(int set_index) {
+  return CalculatorRunner(absl::StrFormat(R"(
+    calculator: "TestGetIntVectorItemCalculator"
+    input_stream: "VECTOR:vector_stream"
+    output_stream: "ITEM:item_stream"
+    options {
+      [mediapipe.GetVectorItemCalculatorOptions.ext] {
+        item_index: %d
+      }
+    }
+  )",
+                                          set_index));
+}
+
+void AddInputVector(CalculatorRunner& runner, const std::vector<int>& inputs,
+                    int timestamp) {
+  runner.MutableInputs()->Tag("VECTOR").packets.push_back(
+      MakePacket<std::vector<int>>(inputs).At(Timestamp(timestamp)));
+}
+
+void AddInputIndex(CalculatorRunner& runner, int index, int timestamp) {
+  runner.MutableInputs()->Tag("INDEX").packets.push_back(
+      MakePacket<int>(index).At(Timestamp(timestamp)));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, EmptyIndexStreamNoOutput) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+  const std::vector<int> inputs = {1, 2, 3};
+
+  AddInputVector(runner, inputs, 1);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_EQ(0, outputs.size());
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, SuccessfulExtractionIndexStream) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+  const std::vector<int> inputs = {1, 2, 3};
+  const int index = 1;
+
+  AddInputVector(runner, inputs, 1);
+  AddInputIndex(runner, index, 1);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, SuccessfulExtractionIndexProto) {
+  const int index = 2;
+  CalculatorRunner runner = MakeRunnerWithOptions(index);
+  const std::vector<int> inputs = {1, 2, 3};
+
+  AddInputVector(runner, inputs, 1);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIsPreferred) {
+  CalculatorRunner runner(R"(
+    calculator: "TestGetIntVectorItemCalculator"
+    input_stream: "VECTOR:vector_stream"
+    input_stream: "INDEX:index_stream"
+    output_stream: "ITEM:item_stream"
+    options {
+      [mediapipe.GetVectorItemCalculatorOptions.ext] {
+        item_index: 2
+      }
+    }
+  )");
+  const std::vector<int> inputs = {1, 2, 3};
+  const int stream_index = 0;
+
+  AddInputVector(runner, inputs, 1);
+  AddInputIndex(runner, stream_index, 1);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[stream_index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, NoStreamNorOptionsExpectFail) {
+  CalculatorRunner runner(R"(
+    calculator: "TestGetIntVectorItemCalculator"
+    input_stream: "VECTOR:vector_stream"
+    output_stream: "ITEM:item_stream"
+  )");
+
+  absl::Status status = runner.Run();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      testing::HasSubstr("kIdx(cc).IsConnected() || options.has_item_index()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail1) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+  const std::vector<int> inputs = {1, 2, 3};
+  const int try_index = -1;
+
+  AddInputVector(runner, inputs, 1);
+  AddInputIndex(runner, try_index, 1);
+
+  absl::Status status = runner.Run();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.message(),
+              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+  const std::vector<int> inputs = {1, 2, 3};
+  const int try_index = 3;
+
+  AddInputVector(runner, inputs, 1);
+  AddInputIndex(runner, try_index, 1);
+
+  absl::Status status = runner.Run();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.message(),
+              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
+  const int try_index = -1;
+  CalculatorRunner runner = MakeRunnerWithOptions(try_index);
+  const std::vector<int> inputs = {1, 2, 3};
+
+  AddInputVector(runner, inputs, 1);
+
+  absl::Status status = runner.Run();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.message(),
+              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
+  const int try_index = 3;
+  CalculatorRunner runner = MakeRunnerWithOptions(try_index);
+  const std::vector<int> inputs = {1, 2, 3};
+
+  AddInputVector(runner, inputs, 1);
+
+  absl::Status status = runner.Run();
+  ASSERT_FALSE(status.ok());
+  EXPECT_THAT(status.message(),
+              testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, IndexStreamTwoTimestamps) {
+  CalculatorRunner runner = MakeRunnerWithStream();
+
+  {
+    const std::vector<int> inputs = {1, 2, 3};
+    const int index = 1;
+    AddInputVector(runner, inputs, 1);
+    AddInputIndex(runner, index, 1);
+  }
+  {
+    const std::vector<int> inputs = {5, 6, 7, 8};
+    const int index = 3;
+    AddInputVector(runner, inputs, 2);
+    AddInputIndex(runner, index, 2);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(2), IntPacket(8)));
+  EXPECT_THAT(outputs,
+              testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, IndexOptionsTwoTimestamps) {
+  const int static_index = 2;
+  CalculatorRunner runner = MakeRunnerWithOptions(static_index);
+
+  {
+    const std::vector<int> inputs = {1, 2, 3};
+    AddInputVector(runner, inputs, 1);
+  }
+  {
+    const std::vector<int> inputs = {5, 6, 7, 8};
+    AddInputVector(runner, inputs, 2);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Tag("ITEM").packets;
+  EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(3), IntPacket(7)));
+  EXPECT_THAT(outputs,
+              testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/graph_profile_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message GraphProfileCalculatorOptions {
   extend mediapipe.CalculatorOptions {
     optional GraphProfileCalculatorOptions ext = 367481815;

mediapipe/calculators/core/merge_to_vector_calculator.cc

@@ -0,0 +1,27 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/calculators/core/merge_to_vector_calculator.h"
+
+#include "mediapipe/framework/formats/image.h"
+
+namespace mediapipe {
+namespace api2 {
+
+typedef MergeToVectorCalculator<mediapipe::Image> MergeImagesToVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(MergeImagesToVectorCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/merge_to_vector_calculator.h

@@ -0,0 +1,58 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+
+namespace mediapipe {
+namespace api2 {
+
+template <typename T>
+class MergeToVectorCalculator : public Node {
+ public:
+  static constexpr typename Input<T>::Multiple kIn{""};
+  static constexpr Output<std::vector<T>> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) {
+    const int input_num = kIn(cc).Count();
+    std::vector<T> output_vector(input_num);
+    std::transform(kIn(cc).begin(), kIn(cc).end(), output_vector.begin(),
+                   [](const auto& elem) -> T { return elem.Get(); });
+    kOut(cc).Send(output_vector);
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_

mediapipe/calculators/core/packet_cloner_calculator.cc

@@ -58,6 +58,7 @@ namespace mediapipe {
 class PacketClonerCalculator : public CalculatorBase {
  public:
   static absl::Status GetContract(CalculatorContract* cc) {
+    cc->SetProcessTimestampBounds(true);
     const Ids ids = GetIds(*cc);
     for (const auto& in_out : ids.inputs_outputs) {
       auto& input = cc->Inputs().Get(in_out.in);
@@ -101,30 +102,30 @@ class PacketClonerCalculator : public CalculatorBase {
       }
     }
 
+    bool has_all_inputs = HasAllInputs();
     // Output according to the TICK signal.
-    if (!cc->Inputs().Get(ids_.tick_id).IsEmpty()) {
-      if (output_only_when_all_inputs_received_) {
-        // Return if one of the input is null.
-        for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
-          if (current_[i].IsEmpty()) {
-            if (output_empty_packets_before_all_inputs_received_) {
-              SetAllNextTimestampBounds(cc);
-            }
-            return absl::OkStatus();
-          }
-        }
-      }
+    if (!cc->Inputs().Get(ids_.tick_id).IsEmpty() &&
+        (has_all_inputs || !output_only_when_all_inputs_received_)) {
       // Output each stream.
       for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
         auto& output = cc->Outputs().Get(ids_.inputs_outputs[i].out);
         if (!current_[i].IsEmpty()) {
-          output.AddPacket(current_[i].At(cc->InputTimestamp()));
-        } else {
-          output.SetNextTimestampBound(
-              cc->InputTimestamp().NextAllowedInStream());
+          output.AddPacket(current_[i].At(
+              cc->Inputs().Get(ids_.tick_id).Value().Timestamp()));
         }
       }
     }
+
+    // Set timestamp bounds according to the TICK signal.
+    bool tick_updated = cc->Inputs().Get(ids_.tick_id).Value().Timestamp() ==
+                        cc->InputTimestamp();
+    bool producing_output = has_all_inputs ||
+                            output_empty_packets_before_all_inputs_received_ ||
+                            !output_only_when_all_inputs_received_;
+    if (tick_updated && producing_output) {
+      SetAllNextTimestampBounds(cc);
+    }
+
     return absl::OkStatus();
   }
 
@@ -165,6 +166,15 @@ class PacketClonerCalculator : public CalculatorBase {
     }
   }
 
+  bool HasAllInputs() {
+    for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
+      if (current_[i].IsEmpty()) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   std::vector<Packet> current_;
   Ids ids_;
   bool output_only_when_all_inputs_received_;

mediapipe/calculators/core/packet_cloner_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message PacketClonerCalculatorOptions {
   extend CalculatorOptions {
     optional PacketClonerCalculatorOptions ext = 258872085;

mediapipe/calculators/core/packet_cloner_calculator_test.cc

@@ -33,6 +33,7 @@ namespace {
 
 using ::testing::ElementsAre;
 using ::testing::Eq;
+using ::testing::IsTrue;
 using ::testing::Value;
 
 MATCHER_P2(IntPacket, value, ts, "") {
@@ -45,6 +46,11 @@ MATCHER_P2(FloatPacket, value, ts, "") {
          Value(arg.Timestamp(), Eq(Timestamp(ts)));
 }
 
+MATCHER_P(EmptyPacket, ts, "") {
+  return Value(arg.IsEmpty(), IsTrue()) &&
+         Value(arg.Timestamp(), Eq(Timestamp(ts)));
+}
+
 template <typename T>
 absl::Status SendPacket(const std::string& input_name, T value, int ts,
                         CalculatorGraph& graph) {
@@ -342,6 +348,105 @@ TEST_P(PacketClonerCalculatorTest,
                       FloatPacket(40.0f, 40000))));
 }
 
+class PacketClonerCalculatorGatedInputTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    CalculatorGraphConfig graph_config =
+        ParseTextProtoOrDie<CalculatorGraphConfig>([&]() {
+          return R"pb(
+            input_stream: 'input'
+            input_stream: 'input_enabled'
+            input_stream: 'tick'
+            input_stream: 'tick_enabled'
+            node {
+              calculator: 'GateCalculator'
+              input_stream: 'tick'
+              input_stream: 'ALLOW:tick_enabled'
+              output_stream: 'tick_gated'
+            }
+            node {
+              calculator: 'GateCalculator'
+              input_stream: 'input'
+              input_stream: 'ALLOW:input_enabled'
+              output_stream: 'input_gated'
+            }
+            node {
+              calculator: 'PacketClonerCalculator'
+              input_stream: 'input_gated'
+              input_stream: 'TICK:tick_gated'
+              output_stream: 'output'
+            })pb";
+        }());
+
+    MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+    MP_ASSERT_OK(graph.ObserveOutputStream(
+        "output",
+        [this](Packet const& packet) {
+          output.push_back(packet);
+          return absl::OkStatus();
+        },
+        true));
+    MP_ASSERT_OK(graph.StartRun({}));
+  }
+
+  CalculatorGraph graph;
+  std::vector<Packet> output;
+};
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+       PropagatesTimestampBoundsWithEmptyInput) {
+  MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/100, graph));
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+  MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+  MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/200, graph));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(output, ElementsAre(EmptyPacket(100)));
+}
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+       PropagatesTimestampBoundsWithInput) {
+  MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/100, graph));
+  MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/100, graph));
+
+  MP_ASSERT_OK(SendPacket("tick_enabled", true, /*ts=*/100, graph));
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+  MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/110, graph));
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/110, graph));
+
+  MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+  MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/200, graph));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(output, ElementsAre(IntPacket(1, 100), EmptyPacket(110)));
+}
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+       PropagatesTimestampBoundsFromTick) {
+  MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/100, graph));
+  MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/100, graph));
+
+  MP_ASSERT_OK(SendPacket("tick_enabled", true, /*ts=*/100, graph));
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+  MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/110, graph));
+  MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/110, graph));
+
+  MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/200, graph));
+  MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/200, graph));
+
+  MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+  MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/200, graph));
+
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  EXPECT_THAT(output, ElementsAre(IntPacket(1, 100), EmptyPacket(200)));
+}
+
 INSTANTIATE_TEST_SUITE_P(PacketClonerCalculator, PacketClonerCalculatorTest,
                          testing::ValuesIn({Params{.use_tick_tag = false},
                                             Params{.use_tick_tag = true}}));

mediapipe/calculators/core/packet_resampler_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message PacketResamplerCalculatorOptions {
   extend CalculatorOptions {
     optional PacketResamplerCalculatorOptions ext = 95743844;

mediapipe/calculators/core/packet_sequencer_calculator.cc

@@ -0,0 +1,103 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <memory>
+
+#include "mediapipe/framework/api2/contract.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// This calculator assigns a timestamp to each "INPUT" packet reflecting
+// the most recent "TICK" timestamp.
+//
+// Each "TICK" timestamp is propagated as a settled "OUTPUT" timestamp.
+// This allows "TICK" packets to be processed right away.
+// When an "INPUT" packet arrives, it is sent to the "OUTPUT" stream with
+// the next unsettled "OUTPUT" timestamp, which is normally one greater than
+// the most recent "TICK" timestamp.
+//
+// If a "TICK" packet and an "INPUT" packet arrive together, the "OUTPUT"
+// packet timestamp is derived from the previous "TICK" timestamp,
+// and the new "OUTPUT" bound is derived from the current "TICK" timestamp.
+// This allows the current "INPUT" packet to cover the current "TICK" timestamp.
+//
+// Example config:
+// node {
+//   calculator: "PacketSequencerCalculator"
+//   input_stream: "INPUT:switch_selection"
+//   input_stream: "TICK:input_image"
+//   input_stream: "TICK:input_audio"
+//   output_stream: "OUTPUT:switch_selection_timed"
+// }
+//
+class PacketSequencerCalculator : public Node {
+ public:
+  static constexpr Input<AnyType>::Multiple kInput{"INPUT"};
+  static constexpr Input<AnyType>::Multiple kTick{"TICK"};
+  static constexpr Output<AnyType>::Multiple kOutput{"OUTPUT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kInput, kTick, kOutput,
+                          StreamHandler("ImmediateInputStreamHandler"),
+                          TimestampChange::Arbitrary());
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_EQ(kInput(cc).Count(), kOutput(cc).Count());
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    // Pass through any input packets at the output stream bound.
+    for (int i = 0; i < kInput(cc).Count(); ++i) {
+      Timestamp stream_bound = kOutput(cc)[i].NextTimestampBound();
+      const PacketBase input_packet = kInput(cc)[i].packet();
+      if (!input_packet.IsEmpty()) {
+        Timestamp output_ts = std::max(Timestamp::Min(), stream_bound);
+        kOutput(cc)[i].Send(input_packet.At(output_ts));
+      }
+    }
+
+    // Find the new tick timestamp, if any.
+    Timestamp tick_ts = Timestamp::Min();
+    for (int i = 0; i < kTick(cc).Count(); ++i) {
+      const PacketBase& tick_packet = kTick(cc)[i].packet();
+      // For either an input packet or an empty input stream,
+      // the packet timestamp indicates the latest "settled timestamp",
+      // and when it arrives it equals the InputTimestamp().
+      if (tick_packet.timestamp() == cc->InputTimestamp()) {
+        tick_ts = std::max(tick_ts, tick_packet.timestamp());
+        break;
+      }
+    }
+
+    // Advance all output stream bounds past the tick timestamp.
+    for (int i = 0; i < kInput(cc).Count(); ++i) {
+      Timestamp stream_bound = kOutput(cc)[i].NextTimestampBound();
+      if (tick_ts >= stream_bound) {
+        kOutput(cc)[i].SetNextTimestampBound(tick_ts.NextAllowedInStream());
+      }
+    }
+    return absl::OkStatus();
+  }
+};
+
+MEDIAPIPE_REGISTER_NODE(PacketSequencerCalculator);
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/core/packet_sequencer_calculator_test.cc

@@ -0,0 +1,118 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+// Returns a CalculatorGraph to run a single calculator.
+CalculatorGraph BuildCalculatorGraph(CalculatorGraphConfig::Node node_config) {
+  CalculatorGraphConfig config;
+  *config.add_node() = node_config;
+  *config.mutable_input_stream() = node_config.input_stream();
+  *config.mutable_output_stream() = node_config.output_stream();
+  *config.mutable_input_side_packet() = node_config.input_side_packet();
+  *config.mutable_output_side_packet() = node_config.output_side_packet();
+  return CalculatorGraph(config);
+}
+
+// Creates a string packet.
+Packet pack(std::string data, int timestamp) {
+  return MakePacket<std::string>(data).At(Timestamp(timestamp));
+}
+
+// Tests showing packet timestamp synchronization through
+// PacketSequencerCalculator.
+class PacketSequencerCalculatorTest : public ::testing::Test {
+ protected:
+  PacketSequencerCalculatorTest() {}
+  ~PacketSequencerCalculatorTest() override {}
+  void SetUp() override {}
+  void TearDown() override {}
+
+  // Defines a PacketSequencerCalculator CalculatorGraphConfig::Node.
+  CalculatorGraphConfig::Node BuildNodeConfig() {
+    CalculatorGraphConfig::Node result;
+    *result.mutable_calculator() = "PacketSequencerCalculator";
+    *result.add_input_stream() = "INPUT:select";
+    *result.add_input_stream() = "TICK:0:frame";
+    *result.add_input_stream() = "TICK:1:mask";
+    *result.add_output_stream() = "OUTPUT:select_timed";
+    return result;
+  }
+};
+
+// Shows the PacketSequencerCalculator is available.
+TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
+  EXPECT_TRUE(
+      CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
+}
+
+// Shows how control packets recieve timestamps before and after frame packets
+// have arrived.
+TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
+  CalculatorGraphConfig::Node node_config = BuildNodeConfig();
+  CalculatorGraph graph = BuildCalculatorGraph(node_config);
+  std::vector<Packet> outputs;
+  MP_ASSERT_OK(graph.ObserveOutputStream("select_timed", [&](const Packet& p) {
+    outputs.push_back(p);
+    return absl::OkStatus();
+  }));
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  // Some control packets arrive.
+  MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t10", 10)));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t20", 20)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // The control packets are assigned low timestamps.
+  ASSERT_EQ(outputs.size(), 2);
+  EXPECT_EQ(outputs[0].Get<std::string>(), "p0_t10");
+  EXPECT_EQ(outputs[0].Timestamp(), Timestamp::Min());
+  EXPECT_EQ(outputs[1].Timestamp(), Timestamp::Min() + 1);
+
+  // Some frame packets arrive.
+  MP_ASSERT_OK(graph.AddPacketToInputStream("mask", pack("p2_t10", 10)));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("frame", pack("p1_t20", 20)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Some more control packets arrive.
+  MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t30", 30)));
+  MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t40", 40)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // New control packets are assigned timestamps following Timestamp(20).
+  ASSERT_EQ(outputs.size(), 4);
+  EXPECT_EQ(outputs[2].Get<std::string>(), "p0_t30");
+  EXPECT_EQ(outputs[2].Timestamp(), Timestamp(21));
+  EXPECT_EQ(outputs[3].Timestamp(), Timestamp(22));
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/packet_thinner_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message PacketThinnerCalculatorOptions {
   extend CalculatorOptions {
     optional PacketThinnerCalculatorOptions ext = 288533508;

mediapipe/calculators/core/quantize_float_vector_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message QuantizeFloatVectorCalculatorOptions {
   extend CalculatorOptions {
     optional QuantizeFloatVectorCalculatorOptions ext = 259848061;

mediapipe/calculators/core/sequence_shift_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 message SequenceShiftCalculatorOptions {
   extend CalculatorOptions {
     optional SequenceShiftCalculatorOptions ext = 107633927;

mediapipe/calculators/core/split_vector_calculator.proto

@@ -18,8 +18,6 @@ package mediapipe;
 
 import "mediapipe/framework/calculator.proto";
 
-option objc_class_prefix = "MediaPipe";
-
 // A Range {begin, end} specifies beginning ane ending indices to splice a
 // vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
 // begin index inclusive and end index exclusive.

mediapipe/calculators/image/scale_image_calculator.cc

@@ -573,8 +573,13 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
       // ImageFrame immediately, before cropping and scaling. Investigate how to
       // make color space conversion more efficient when cropping or scaling is
       // also needed.
-      image_frame_util::YUVImageToImageFrame(*yuv_image, &converted_image_frame,
-                                             options_.use_bt709());
+      if (options_.use_bt709() || yuv_image->fourcc() == libyuv::FOURCC_ANY) {
+        image_frame_util::YUVImageToImageFrame(
+            *yuv_image, &converted_image_frame, options_.use_bt709());
+      } else {
+        image_frame_util::YUVImageToImageFrameFromFormat(
+            *yuv_image, &converted_image_frame);
+      }
       image_frame = &converted_image_frame;
     } else if (output_format_ == ImageFormat::YCBCR420P) {
       RET_CHECK(row_start_ == 0 && col_start_ == 0 &&

mediapipe/calculators/tensor/BUILD

@@ -153,11 +153,12 @@ cc_library(
     tags = ["nomac"],  # config problem with cpuinfo via TF
     visibility = ["//visibility:public"],
     deps = [
+        ":inference_calculator_cc_proto",
         ":inference_calculator_interface",
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/gpu:gl_calculator_helper",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
-        "@org_tensorflow//tensorflow/lite:framework_stable",
         "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
     ],
     alwayslink = 1,
@@ -172,6 +173,7 @@ cc_library(
         ":inference_calculator_interface",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/gpu:gl_calculator_helper",
         "//mediapipe/util/tflite:tflite_gpu_runner",
@@ -231,6 +233,7 @@ cc_library(
     deps = [
         ":inference_calculator_interface",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
         "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
         "@org_tensorflow//tensorflow/lite:framework_stable",
         "@org_tensorflow//tensorflow/lite/c:c_api_types",
@@ -636,6 +639,7 @@ cc_library(
         ":image_to_tensor_calculator_cc_proto",
         ":image_to_tensor_converter",
         ":image_to_tensor_utils",
+        ":loose_headers",
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:image_frame",
@@ -990,3 +994,58 @@ cc_library(
     }),
     alwayslink = 1,
 )
+
+cc_library(
+    name = "tensors_dequantization_calculator",
+    srcs = ["tensors_dequantization_calculator.cc"],
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+# For a more maintainable build this target should not exist and the headers
+# should  be split into the existing cc_library targets, but this change was
+# automatically  done so that we can remove long standing issues and complexity
+# in the build system. It's up to the OWNERS of this package to get rid of it or
+# not. The use of the textual_hdrs attribute is discouraged, use hdrs instead.
+# Here it is used to avoid header parsing errors in packages where the feature
+# parse_headers was enabled since loose headers were not being parsed.
+cc_library(
+    name = "loose_headers",
+    tags = ["avoid_dep"],
+    textual_hdrs = [
+        "image_to_tensor_converter_gl_buffer.h",
+        "image_to_tensor_converter_gl_texture.h",
+    ],
+    visibility = [":__pkg__"],
+)
+
+cc_test(
+    name = "tensors_dequantization_calculator_test",
+    srcs = ["tensors_dequantization_calculator_test.cc"],
+    deps = [
+        ":tensors_dequantization_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/status",
+    ],
+)

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -56,14 +56,14 @@ namespace api2 {
 // previous output.
 //
 // The calculator has two running modes:
-//   Streaming mode: when "streaming_mode" is set to true in the calculator
+//   Streaming mode: when "stream_mode" is set to true in the calculator
 //     options, the calculator treats the input audio stream as a continuous
 //     stream. Thus, any samples that are not consumed in the previous runs will
 //     be cached in a global sample buffer. The audio data resampled from the
 //     current raw audio input will be appended to the global sample buffer.
 //     The calculator will process the global sample buffer and output as many
 //     tensors as possible.
-//   Non-streaming mode: when "streaming_mode" is set to false in the calculator
+//   Non-streaming mode: when "stream_mode" is set to false in the calculator
 //     options, the calculators treats the packets in the input audio stream as
 //     a batch of unrelated audio buffers. In each Process() call, the input
 //     buffer will be frist resampled, and framed as fixed-sized, possibly
@@ -104,7 +104,7 @@ namespace api2 {
 //       num_samples: 512
 //       num_overlapping_samples: 64
 //       target_sample_rate: 16000
-//       streaming_mode: true # or false
+//       stream_mode: true # or false
 //     }
 //   }
 // }
@@ -136,7 +136,7 @@ class AudioToTensorCalculator : public Node {
   // The number of samples per channel to advance after the current frame is
   // processed.
   int frame_step_;
-  bool streaming_mode_;
+  bool stream_mode_;
   bool check_inconsistent_timestamps_;
   Timestamp initial_timestamp_ = Timestamp::Unstarted();
   int64 cumulative_input_samples_ = 0;
@@ -151,8 +151,9 @@ class AudioToTensorCalculator : public Node {
   Matrix sample_buffer_;
   int processed_buffer_cols_ = 0;
 
-  absl::Status ProcessStreamingData(CalculatorContext* cc);
-  absl::Status ProcessNonStreamingData(CalculatorContext* cc);
+  absl::Status ProcessStreamingData(CalculatorContext* cc, const Matrix& input);
+  absl::Status ProcessNonStreamingData(CalculatorContext* cc,
+                                       const Matrix& input);
 
   absl::Status SetupStreamingResampler(double input_sample_rate_);
   void AppendToSampleBuffer(Matrix buffer_to_append);
@@ -172,7 +173,7 @@ absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
         "AudioToTensorCalculatorOptions must specifiy "
         "`num_channels`, `num_samples`, and `target_sample_rate`.");
   }
-  if (options.streaming_mode()) {
+  if (options.stream_mode()) {
     // Explicitly disables tiemstamp offset to disallow the timestamp bound
     // from the input streams to be propagated to the output streams.
     // In the streaming mode, the output timestamp bound is based on
@@ -196,8 +197,8 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
     frame_step_ = num_samples_;
   }
   target_sample_rate_ = options.target_sample_rate();
-  streaming_mode_ = options.streaming_mode();
-  if (streaming_mode_) {
+  stream_mode_ = options.stream_mode();
+  if (stream_mode_) {
     check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
     sample_buffer_.resize(num_channels_, Eigen::NoChange);
   }
@@ -210,7 +211,7 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
     mediapipe::TimeSeriesHeader input_header;
     MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
         kAudioIn(cc).Header(), &input_header));
-    if (streaming_mode_) {
+    if (stream_mode_) {
       MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
     } else {
       source_sample_rate_ = input_header.sample_rate();
@@ -223,7 +224,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
   if (cc->InputTimestamp() == Timestamp::PreStream()) {
     double current_source_sample_rate = kAudioSampleRateIn(cc).Get();
     if (cc->Options<mediapipe::AudioToTensorCalculatorOptions>()
-            .streaming_mode()) {
+            .stream_mode()) {
       return SetupStreamingResampler(current_source_sample_rate);
     } else {
       source_sample_rate_ = current_source_sample_rate;
@@ -232,21 +233,28 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
   }
   // Sanity checks.
   const auto& input_frame = kAudioIn(cc).Get();
-  if (input_frame.rows() != num_channels_) {
+  const bool channels_match = input_frame.rows() == num_channels_;
+  // The special case of `num_channels_ == 1` is automatic mixdown to mono.
+  const bool mono_output = num_channels_ == 1;
+  if (!mono_output && !channels_match) {
     return absl::InvalidArgumentError(absl::StrFormat(
         "Audio input has %d channel(s) but the model requires %d channel(s).",
         input_frame.rows(), num_channels_));
   }
-  if (num_channels_ > 1 && input_frame.IsRowMajor) {
+  if (!mono_output && input_frame.IsRowMajor) {
     return absl::InvalidArgumentError(
         "The audio data should be stored in column-major.");
   }
-  return streaming_mode_ ? ProcessStreamingData(cc)
-                         : ProcessNonStreamingData(cc);
+  CHECK(channels_match || mono_output);
+  const Matrix& input = channels_match ? input_frame
+                                       // Mono mixdown.
+                                       : input_frame.colwise().mean();
+  return stream_mode_ ? ProcessStreamingData(cc, input)
+                      : ProcessNonStreamingData(cc, input);
 }
 
 absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
-  if (!streaming_mode_) {
+  if (!stream_mode_) {
     return absl::OkStatus();
   }
   if (resampler_) {
@@ -258,8 +266,8 @@ absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
 }
 
 absl::Status AudioToTensorCalculator::ProcessStreamingData(
-    CalculatorContext* cc) {
-  const auto& input_buffer = kAudioIn(cc).Get();
+    CalculatorContext* cc, const Matrix& input) {
+  const auto& input_buffer = input;
   if (initial_timestamp_ == Timestamp::Unstarted()) {
     initial_timestamp_ = cc->InputTimestamp();
     next_output_timestamp_ = initial_timestamp_;
@@ -303,10 +311,10 @@ absl::Status AudioToTensorCalculator::ProcessStreamingData(
 }
 
 absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
-    CalculatorContext* cc) {
+    CalculatorContext* cc, const Matrix& input) {
   initial_timestamp_ = cc->InputTimestamp();
   next_output_timestamp_ = initial_timestamp_;
-  const auto& input_frame = kAudioIn(cc).Get();
+  const auto& input_frame = input;
   double source_sample_rate = kAudioSampleRateIn(cc).GetOr(source_sample_rate_);
 
   if (source_sample_rate != -1 && source_sample_rate != target_sample_rate_) {
@@ -362,7 +370,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
                                                     CalculatorContext* cc) {
   int next_frame_first_col = 0;
   std::vector<Timestamp> timestamps;
-  while ((!streaming_mode_ || !should_flush) &&
+  while ((!stream_mode_ || !should_flush) &&
          next_frame_first_col + num_samples_ <= buffer.cols()) {
     ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
                                              0, next_frame_first_col,
@@ -383,7 +391,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
     // Timestamp::Max() will be emitted. In the non-streaming mode, each
     // Process() invocation will process the entire buffer completely.
     Timestamp timestamp =
-        streaming_mode_ ? Timestamp::Max() : next_output_timestamp_;
+        stream_mode_ ? Timestamp::Max() : next_output_timestamp_;
     timestamps.push_back(timestamp);
     kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
   }

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -24,6 +24,7 @@ message AudioToTensorCalculatorOptions {
   }
 
   // The required number of channels the output audio tensor has.
+  // If set to 1, multichannel signals will be automatically mixed down to mono.
   optional int64 num_channels = 1;
 
   // The required number of samples per channel the output audio tensor has.
@@ -38,7 +39,7 @@ message AudioToTensorCalculatorOptions {
 
   // Whether to treat the input audio stream as a continous stream or a batch
   // of unrelated audio buffers.
-  optional bool streaming_mode = 5 [default = true];
+  optional bool stream_mode = 5 [default = true];
 
   // Set to false to disable checks for jitter in timestamp values. Useful with
   // live audio input.

mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc

@@ -63,7 +63,10 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
  protected:
   void SetUp() override {}
   void Run(int num_samples, int num_overlapping_samples,
-           double resampling_factor, const Matrix& input_matrix) {
+           double resampling_factor, const Matrix& input_matrix,
+           int num_channels_override = 0) {
+    const int num_channels = num_channels_override == 0 ? input_matrix.rows()
+                                                        : num_channels_override;
     double input_sample_rate = 10000;
     double target_sample_rate = input_sample_rate * resampling_factor;
     auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
@@ -84,12 +87,12 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
               num_samples: $1
               num_overlapping_samples: $2
               target_sample_rate: $3
-              streaming_mode: false
+              stream_mode: false
             }
           }
         }
         )",
-                         /*$0=*/input_matrix.rows(),
+                         /*$0=*/num_channels,
                          /*$1=*/num_samples, /*$2=*/num_overlapping_samples,
                          /*$3=*/target_sample_rate));
     tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
@@ -114,20 +117,21 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
   }
 
   void CheckTensorsOutputPackets(const Matrix& expected_matrix,
-                                 int sample_offset, int num_tensors_per_input) {
+                                 int sample_offset, int num_tensors_per_input,
+                                 bool mono = false) {
     ASSERT_EQ(num_iterations_ * num_tensors_per_input, tensors_packets_.size());
     for (int i = 0; i < num_iterations_; ++i) {
       for (int j = 0; j < num_tensors_per_input; ++j) {
         CheckTensorsOutputPacket(
             expected_matrix, tensors_packets_[i * num_tensors_per_input + j],
-            /*sample_offset*/ sample_offset * j, /*index=*/j);
+            /*sample_offset=*/sample_offset * j, /*index=*/j, /*mono=*/mono);
       }
     }
   }
 
   void CheckTensorsOutputPacket(const Matrix& expected_matrix,
                                 const Packet& packet, int sample_offset,
-                                int index) {
+                                int index, bool mono = false) {
     MP_ASSERT_OK(packet.ValidateAsType<std::vector<Tensor>>());
     ASSERT_EQ(1, packet.Get<std::vector<Tensor>>().size());
     const Tensor& output_tensor = packet.Get<std::vector<Tensor>>()[0];
@@ -137,7 +141,11 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
     for (int i = 0; i < num_values; ++i) {
       if (i + sample_offset >= expected_matrix.size()) {
         EXPECT_FLOAT_EQ(output_floats[i], 0);
+      } else if (mono) {
+        EXPECT_FLOAT_EQ(output_floats[i],
+                        expected_matrix.coeff(0, i + sample_offset));
       } else {
+        // Stereo.
         EXPECT_FLOAT_EQ(output_floats[i],
                         expected_matrix.coeff((i + sample_offset) % 2,
                                               (i + sample_offset) / 2))
@@ -209,6 +217,17 @@ TEST_F(AudioToTensorCalculatorNonStreamingModeTest, TensorsWithZeroPadding) {
   CloseGraph();
 }
 
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Mixdown) {
+  auto input_matrix = CreateTestMatrix(2, 8, 0);
+  Run(/*num_samples=*/4, /*num_overlapping_samples=*/2,
+      /*resampling_factor=*/1.0f, *input_matrix, /*num_channels_override=*/1);
+  const Matrix& mono_matrix = input_matrix->colwise().mean();
+  CheckTensorsOutputPackets(mono_matrix, /*sample_offset=*/2,
+                            /*num_tensors_per_input=*/4, /*mono=*/true);
+  CheckTimestampsOutputPackets({0, 200, 400, 600});
+  CloseGraph();
+}
+
 TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Downsampling) {
   auto input_matrix = CreateTestMatrix(2, 1024, 0);
   Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
@@ -299,7 +318,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
               num_samples: $0
               num_overlapping_samples: $1
               target_sample_rate: $2
-              streaming_mode:true
+              stream_mode:true
             }
           }
         }

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -348,14 +348,13 @@ class ImageToTensorCalculator : public Node {
                          CreateImageToGlBufferTensorConverter(
                              cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
 #else
-        // Check whether the underlying storage object is a GL texture.
-        if (image.GetGpuBuffer()
-                .internal_storage<mediapipe::GlTextureBuffer>()) {
+        if (!gpu_converter_) {
           ASSIGN_OR_RETURN(
               gpu_converter_,
               CreateImageToGlTextureTensorConverter(
                   cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
-        } else {
+        }
+        if (!gpu_converter_) {
           return absl::UnimplementedError(
               "ImageToTensorConverter for the input GPU image is unavailable.");
         }

mediapipe/calculators/tensor/inference_calculator.cc

@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+#include "absl/status/status.h"
 #include "absl/strings/string_view.h"
 #include "mediapipe/calculators/tensor/inference_calculator.pb.h"
 #include "mediapipe/framework/api2/packet.h"

mediapipe/calculators/tensor/inference_calculator.h

@@ -20,18 +20,13 @@
 #include <string>
 #include <vector>
 
-#include "absl/memory/memory.h"
 #include "mediapipe/calculators/tensor/inference_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/tensor.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/util/tflite/tflite_model_loader.h"
 #include "tensorflow/lite/core/api/op_resolver.h"
-#include "tensorflow/lite/error_reporter.h"
-#include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/model.h"
 
 namespace mediapipe {
 namespace api2 {
@@ -119,10 +114,10 @@ class InferenceCalculator : public NodeIntf {
   using TfLiteDelegatePtr =
       std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
 
-  absl::StatusOr<Packet<TfLiteModelPtr>> GetModelAsPacket(
+  static absl::StatusOr<Packet<TfLiteModelPtr>> GetModelAsPacket(
       CalculatorContext* cc);
 
-  absl::StatusOr<Packet<tflite::OpResolver>> GetOpResolverAsPacket(
+  static absl::StatusOr<Packet<tflite::OpResolver>> GetOpResolverAsPacket(
       CalculatorContext* cc);
 };
 

mediapipe/calculators/tensor/inference_calculator_cpu.cc

@@ -12,13 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
 #include <cstring>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "absl/status/status.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/interpreter_builder.h"
 #if defined(MEDIAPIPE_ANDROID)
 #include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
@@ -63,9 +66,9 @@ int GetXnnpackNumThreads(
 }
 
 template <typename T>
-void CopyTensorBuffer(const Tensor& input_tensor,
-                      tflite::Interpreter* interpreter,
-                      int input_tensor_index) {
+void CopyTensorBufferToInterpreter(const Tensor& input_tensor,
+                                   tflite::Interpreter* interpreter,
+                                   int input_tensor_index) {
   auto input_tensor_view = input_tensor.GetCpuReadView();
   auto input_tensor_buffer = input_tensor_view.buffer<T>();
   T* local_tensor_buffer =
@@ -73,6 +76,18 @@ void CopyTensorBuffer(const Tensor& input_tensor,
   std::memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor.bytes());
 }
 
+template <typename T>
+void CopyTensorBufferFromInterpreter(tflite::Interpreter* interpreter,
+                                     int output_tensor_index,
+                                     Tensor* output_tensor) {
+  auto output_tensor_view = output_tensor->GetCpuWriteView();
+  auto output_tensor_buffer = output_tensor_view.buffer<T>();
+  T* local_tensor_buffer =
+      interpreter->typed_output_tensor<T>(output_tensor_index);
+  std::memcpy(output_tensor_buffer, local_tensor_buffer,
+              output_tensor->bytes());
+}
+
 }  // namespace
 
 class InferenceCalculatorCpuImpl
@@ -99,7 +114,7 @@ class InferenceCalculatorCpuImpl
 
 absl::Status InferenceCalculatorCpuImpl::UpdateContract(
     CalculatorContract* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
   RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
       << "Either model as side packet or model path in options is required.";
 
@@ -118,20 +133,32 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
   RET_CHECK(!input_tensors.empty());
   auto output_tensors = absl::make_unique<std::vector<Tensor>>();
 
+  if (input_tensor_type_ == kTfLiteNoType) {
+    input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
+  }
+
   // Read CPU input into tensors.
   for (int i = 0; i < input_tensors.size(); ++i) {
     switch (input_tensor_type_) {
       case TfLiteType::kTfLiteFloat16:
       case TfLiteType::kTfLiteFloat32: {
-        CopyTensorBuffer<float>(input_tensors[i], interpreter_.get(), i);
+        CopyTensorBufferToInterpreter<float>(input_tensors[i],
+                                             interpreter_.get(), i);
         break;
       }
       case TfLiteType::kTfLiteUInt8: {
-        CopyTensorBuffer<uint8>(input_tensors[i], interpreter_.get(), i);
+        CopyTensorBufferToInterpreter<uint8>(input_tensors[i],
+                                             interpreter_.get(), i);
         break;
       }
       case TfLiteType::kTfLiteInt8: {
-        CopyTensorBuffer<int8>(input_tensors[i], interpreter_.get(), i);
+        CopyTensorBufferToInterpreter<int8>(input_tensors[i],
+                                            interpreter_.get(), i);
+        break;
+      }
+      case TfLiteType::kTfLiteInt32: {
+        CopyTensorBufferToInterpreter<int32_t>(input_tensors[i],
+                                               interpreter_.get(), i);
         break;
       }
       default:
@@ -148,13 +175,41 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
   output_tensors->reserve(tensor_indexes.size());
   for (int i = 0; i < tensor_indexes.size(); ++i) {
     TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
-    output_tensors->emplace_back(
-        Tensor::ElementType::kFloat32,
-        Tensor::Shape{std::vector<int>{
-            tensor->dims->data, tensor->dims->data + tensor->dims->size}});
-    auto cpu_view = output_tensors->back().GetCpuWriteView();
-    std::memcpy(cpu_view.buffer<float>(), tensor->data.f,
-                output_tensors->back().bytes());
+    Tensor::Shape shape{std::vector<int>{
+        tensor->dims->data, tensor->dims->data + tensor->dims->size}};
+    switch (tensor->type) {
+      case TfLiteType::kTfLiteFloat16:
+      case TfLiteType::kTfLiteFloat32:
+        output_tensors->emplace_back(Tensor::ElementType::kFloat32, shape);
+        CopyTensorBufferFromInterpreter<float>(interpreter_.get(), i,
+                                               &output_tensors->back());
+        break;
+      case TfLiteType::kTfLiteUInt8:
+        output_tensors->emplace_back(
+            Tensor::ElementType::kUInt8, shape,
+            Tensor::QuantizationParameters{tensor->params.scale,
+                                           tensor->params.zero_point});
+        CopyTensorBufferFromInterpreter<uint8>(interpreter_.get(), i,
+                                               &output_tensors->back());
+        break;
+      case TfLiteType::kTfLiteInt8:
+        output_tensors->emplace_back(
+            Tensor::ElementType::kInt8, shape,
+            Tensor::QuantizationParameters{tensor->params.scale,
+                                           tensor->params.zero_point});
+        CopyTensorBufferFromInterpreter<int8>(interpreter_.get(), i,
+                                              &output_tensors->back());
+        break;
+      case TfLiteType::kTfLiteInt32:
+        output_tensors->emplace_back(Tensor::ElementType::kInt32, shape);
+        CopyTensorBufferFromInterpreter<int32_t>(interpreter_.get(), i,
+                                                 &output_tensors->back());
+        break;
+      default:
+        return absl::InvalidArgumentError(
+            absl::StrCat("Unsupported output tensor type:",
+                         TfLiteTypeGetName(tensor->type)));
+    }
   }
   kOutTensors(cc).Send(std::move(output_tensors));
   return absl::OkStatus();
@@ -188,7 +243,6 @@ absl::Status InferenceCalculatorCpuImpl::InitInterpreter(
 
 absl::Status InferenceCalculatorCpuImpl::AllocateTensors() {
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
-  input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
   return absl::OkStatus();
 }
 
@@ -198,13 +252,14 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(
       cc->Options<mediapipe::InferenceCalculatorOptions>();
   auto opts_delegate = calculator_opts.delegate();
   if (!kDelegate(cc).IsEmpty()) {
-    mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
-        kDelegate(cc).Get();
-    CHECK(input_side_packet_delegate.has_tflite() ||
-          input_side_packet_delegate.has_xnnpack() ||
-          input_side_packet_delegate.has_nnapi() ||
-          input_side_packet_delegate.delegate_case() ==
-              mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+    const mediapipe::InferenceCalculatorOptions::Delegate&
+        input_side_packet_delegate = kDelegate(cc).Get();
+    RET_CHECK(
+        input_side_packet_delegate.has_tflite() ||
+        input_side_packet_delegate.has_xnnpack() ||
+        input_side_packet_delegate.has_nnapi() ||
+        input_side_packet_delegate.delegate_case() ==
+            mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
         << "inference_calculator_cpu only supports delegate input side packet "
         << "for TFLite, XNNPack and Nnapi";
     opts_delegate.MergeFrom(input_side_packet_delegate);

mediapipe/calculators/tensor/inference_calculator_gl.cc

@@ -15,11 +15,14 @@
 #include <cstring>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "tensorflow/lite/delegates/gpu/gl_delegate.h"
 
@@ -36,111 +39,64 @@ class InferenceCalculatorGlImpl
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  absl::Status LoadModel(CalculatorContext* cc);
-  absl::Status LoadDelegate(CalculatorContext* cc);
-  absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
+  // Helper class that wraps everything related to GPU inference acceleration.
+  class GpuInferenceRunner {
+   public:
+    ~GpuInferenceRunner();
 
-  // TfLite requires us to keep the model alive as long as the interpreter is.
-  Packet<TfLiteModelPtr> model_packet_;
+    absl::Status Init(CalculatorContext* cc,
+                      const mediapipe::InferenceCalculatorOptions::Delegate&
+                          delegate_options);
+    absl::Status LoadModel(CalculatorContext* cc);
+    absl::Status LoadDelegate(
+        CalculatorContext* cc,
+        const mediapipe::InferenceCalculatorOptions::Delegate&
+            delegate_options);
+    absl::Status LoadDelegateAndAllocateTensors(
+        CalculatorContext* cc,
+        const mediapipe::InferenceCalculatorOptions::Delegate&
+            delegate_options);
+    absl::Status Process(CalculatorContext* cc,
+                         const std::vector<Tensor>& input_tensors,
+                         std::vector<Tensor>& output_tensors);
 
-  mediapipe::GlCalculatorHelper gpu_helper_;
-  bool allow_precision_loss_ = false;
+   private:
+    // TfLite requires us to keep the model alive as long as the interpreter is.
+    Packet<TfLiteModelPtr> model_packet_;
+    mediapipe::GlCalculatorHelper gpu_helper_;
+    TfLiteDelegatePtr delegate_;
+    std::unique_ptr<tflite::Interpreter> interpreter_;
+    std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
+    std::vector<std::unique_ptr<Tensor>> gpu_buffers_out_;
+    size_t output_size_ = 0;
+  };
 
-  TfLiteDelegatePtr delegate_;
-  std::unique_ptr<tflite::Interpreter> interpreter_;
-
-  std::vector<Tensor::Shape> output_shapes_;
-  std::vector<std::unique_ptr<Tensor>> gpu_buffers_in_;
-  std::vector<std::unique_ptr<Tensor>> gpu_buffers_out_;
+  std::unique_ptr<GpuInferenceRunner> gpu_inference_runner_;
 };
 
-absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
-  RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
-      << "Either model as side packet or model path in options is required.";
-
-  return mediapipe::GlCalculatorHelper::UpdateContract(cc);
-}
-
-absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
-  mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
-  if (!kDelegate(cc).IsEmpty()) {
-    mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
-        kDelegate(cc).Get();
-    CHECK(input_side_packet_delegate.has_gpu() ||
-          input_side_packet_delegate.delegate_case() ==
-              mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
-        << "inference_calculator_gl only supports delegate input side packet "
-        << "for Gpu";
-    delegate.MergeFrom(input_side_packet_delegate);
-  }
-
-  MP_RETURN_IF_ERROR(LoadModel(cc));
-  MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-  return gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
-    return LoadDelegateAndAllocateTensors(cc);
-  });
-}
-
-absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
-  if (kInTensors(cc).IsEmpty()) {
-    return absl::OkStatus();
-  }
-  const auto& input_tensors = *kInTensors(cc);
-  RET_CHECK(!input_tensors.empty());
-  auto output_tensors = absl::make_unique<std::vector<Tensor>>();
-
-  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-      [this, &input_tensors]() -> ::mediapipe::Status {
-        // Explicitly copy input.
-        for (int i = 0; i < input_tensors.size(); ++i) {
-          glBindBuffer(GL_COPY_READ_BUFFER,
-                       input_tensors[i].GetOpenGlBufferReadView().name());
-          glBindBuffer(GL_COPY_WRITE_BUFFER,
-                       gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
-          glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
-                              input_tensors[i].bytes());
-        }
-        return absl::OkStatus();
-      }));
-
-  // Run inference.
-  RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
-
-  MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
-      [this, &output_tensors]() -> ::mediapipe::Status {
-        output_tensors->reserve(output_shapes_.size());
-        for (int i = 0; i < output_shapes_.size(); ++i) {
-          const auto& t = gpu_buffers_out_[i];
-          output_tensors->emplace_back(Tensor::ElementType::kFloat32,
-                                       gpu_buffers_out_[i]->shape());
-          auto read_view = t->GetOpenGlBufferReadView();
-          glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
-          auto write_view = output_tensors->back().GetOpenGlBufferWriteView();
-          glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
-          glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
-                              t->bytes());
-        }
-        return absl::OkStatus();
-      }));
-
-  kOutTensors(cc).Send(std::move(output_tensors));
-  return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
-  return gpu_helper_.RunInGlContext([this]() -> absl::Status {
+InferenceCalculatorGlImpl::GpuInferenceRunner::~GpuInferenceRunner() {
+  gpu_helper_.RunInGlContext([this]() {
     gpu_buffers_in_.clear();
     gpu_buffers_out_.clear();
     // Delegate must outlive the interpreter, hence the order is important.
     interpreter_ = nullptr;
     delegate_ = nullptr;
-    return absl::OkStatus();
   });
 }
 
-absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Init(
+    CalculatorContext* cc,
+    const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
+  MP_RETURN_IF_ERROR(LoadModel(cc));
+  MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+  return gpu_helper_.RunInGlContext(
+      [this, &cc, &delegate_options]() -> absl::Status {
+        return LoadDelegateAndAllocateTensors(cc, delegate_options);
+      });
+}
+
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadModel(
+    CalculatorContext* cc) {
   ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
   const auto& model = *model_packet_.Get();
   if (kSideInOpResolver(cc).IsConnected()) {
@@ -160,9 +116,11 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
-absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
-    CalculatorContext* cc) {
-  MP_RETURN_IF_ERROR(LoadDelegate(cc));
+absl::Status
+InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegateAndAllocateTensors(
+    CalculatorContext* cc,
+    const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
+  MP_RETURN_IF_ERROR(LoadDelegate(cc, delegate_options));
 
   // AllocateTensors() can be called only after ModifyGraphWithDelegate.
   RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
@@ -173,11 +131,16 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
   return absl::OkStatus();
 }
 
-absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
+    CalculatorContext* cc,
+    const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
   // Configure and create the delegate.
   TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
   options.compile_options.precision_loss_allowed =
-      allow_precision_loss_ ? 1 : 0;
+      (delegate_options.has_gpu() &&
+       delegate_options.gpu().allow_precision_loss())
+          ? 1
+          : 0;
   options.compile_options.preferred_gl_object_type =
       TFLITE_GL_OBJECT_TYPE_FASTEST;
   options.compile_options.dynamic_batch_enabled = 0;
@@ -202,9 +165,9 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
   interpreter_->SetAllowBufferHandleOutput(true);
   // Get output image sizes.
   const auto& output_indices = interpreter_->outputs();
-  output_shapes_.resize(output_indices.size());
+  output_size_ = output_indices.size();
   // Create and bind output buffers.
-  for (int i = 0; i < output_shapes_.size(); ++i) {
+  for (int i = 0; i < output_size_; ++i) {
     const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
     gpu_buffers_out_.emplace_back(absl::make_unique<Tensor>(
         Tensor::ElementType::kFloat32,
@@ -224,5 +187,89 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
+    std::vector<Tensor>& output_tensors) {
+  return gpu_helper_.RunInGlContext(
+      [this, &input_tensors, &output_tensors]() -> absl::Status {
+        // Explicitly copy input.
+        for (int i = 0; i < input_tensors.size(); ++i) {
+          glBindBuffer(GL_COPY_READ_BUFFER,
+                       input_tensors[i].GetOpenGlBufferReadView().name());
+          glBindBuffer(GL_COPY_WRITE_BUFFER,
+                       gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
+          glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
+                              input_tensors[i].bytes());
+        }
+
+        // Run inference.
+        RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+
+        output_tensors.reserve(output_size_);
+        for (int i = 0; i < output_size_; ++i) {
+          const auto& t = gpu_buffers_out_[i];
+          output_tensors.emplace_back(Tensor::ElementType::kFloat32,
+                                      gpu_buffers_out_[i]->shape());
+          auto read_view = t->GetOpenGlBufferReadView();
+          glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
+          auto write_view = output_tensors.back().GetOpenGlBufferWriteView();
+          glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
+          glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
+                              t->bytes());
+        }
+
+        return absl::OkStatus();
+      });
+}
+
+absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
+  RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
+      << "Either model as side packet or model path in options is required.";
+
+  return mediapipe::GlCalculatorHelper::UpdateContract(cc);
+}
+
+absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
+  mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
+  if (!kDelegate(cc).IsEmpty()) {
+    const mediapipe::InferenceCalculatorOptions::Delegate&
+        input_side_packet_delegate = kDelegate(cc).Get();
+    RET_CHECK(
+        (input_side_packet_delegate.has_gpu() &&
+         !input_side_packet_delegate.gpu().use_advanced_gpu_api()) ||
+        input_side_packet_delegate.delegate_case() ==
+            mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+        << "inference_calculator_gl only supports delegate input side packet "
+        << "for Gpu (non advanced)";
+    delegate.MergeFrom(input_side_packet_delegate);
+  }
+
+  gpu_inference_runner_ = std::make_unique<GpuInferenceRunner>();
+  return gpu_inference_runner_->Init(cc, delegate);
+}
+
+absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
+  if (kInTensors(cc).IsEmpty()) {
+    return absl::OkStatus();
+  }
+
+  const auto& input_tensors = *kInTensors(cc);
+  RET_CHECK(!input_tensors.empty());
+  auto output_tensors = absl::make_unique<std::vector<Tensor>>();
+
+  MP_RETURN_IF_ERROR(
+      gpu_inference_runner_->Process(cc, input_tensors, *output_tensors));
+
+  kOutTensors(cc).Send(std::move(output_tensors));
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
+  gpu_inference_runner_ = nullptr;
+  return absl::OkStatus();
+}
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc

@@ -15,10 +15,12 @@
 #include <cstring>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "absl/memory/memory.h"
 #include "absl/status/status.h"
+#include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/inference_calculator.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/util/tflite/tflite_gpu_runner.h"
@@ -28,7 +30,7 @@
 #include "mediapipe/util/android/file/base/file.h"
 #include "mediapipe/util/android/file/base/filesystem.h"
 #include "mediapipe/util/android/file/base/helpers.h"
-#endif  // ANDROID
+#endif  // MEDIAPIPE_ANDROID
 
 namespace mediapipe {
 namespace api2 {
@@ -56,85 +58,71 @@ class InferenceCalculatorGlAdvancedImpl
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  absl::Status ReadGpuCaches();
-  absl::Status SaveGpuCaches();
-  absl::Status InitTFLiteGPURunner(CalculatorContext* cc);
+  // Helper class that saves binary data to disk, or read from disk.
+  class OnDiskCacheHelper {
+   public:
+    absl::Status Init(
+        const mediapipe::InferenceCalculatorOptions& options,
+        const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+            gpu_delegate_options);
+    absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
+    absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
 
-  // TfLite requires us to keep the model alive as long as the interpreter is.
-  Packet<TfLiteModelPtr> model_packet_;
+   private:
+    bool use_kernel_caching_ = false;
+    std::string cached_kernel_filename_;
+    bool use_serialized_model_ = false;
+    std::string serialized_model_path_;
+  };
 
-  mediapipe::GlCalculatorHelper gpu_helper_;
-  std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
-  bool allow_precision_loss_ = false;
-  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
-      tflite_gpu_runner_api_;
-  mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
-      tflite_gpu_runner_usage_;
+  // Helper class that wraps everything related to GPU inference acceleration.
+  class GpuInferenceRunner {
+   public:
+    absl::Status Init(
+        CalculatorContext* cc,
+        const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
 
-  std::vector<Tensor::Shape> output_shapes_;
+    absl::StatusOr<std::vector<Tensor>> Process(
+        const std::vector<Tensor>& input_tensors);
 
-  bool use_kernel_caching_ = false;
-  std::string cached_kernel_filename_;
-  bool use_serialized_model_ = false;
-  std::string serialized_model_path_;
+    absl::Status Close();
+
+   private:
+    absl::Status InitTFLiteGPURunner(
+        CalculatorContext* cc,
+        const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
+
+    // TfLite requires us to keep the model alive as long as the interpreter is.
+    Packet<TfLiteModelPtr> model_packet_;
+
+    mediapipe::GlCalculatorHelper gpu_helper_;
+    std::unique_ptr<tflite::gpu::TFLiteGPURunner> tflite_gpu_runner_;
+
+    std::vector<Tensor::Shape> output_shapes_;
+
+    OnDiskCacheHelper on_disk_cache_helper_;
+  };
+
+  std::unique_ptr<GpuInferenceRunner> gpu_inference_runner_;
 };
 
-absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
-    CalculatorContract* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
-  RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
-      << "Either model as side packet or model path in options is required.";
-
-  MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
-  return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
-  mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
-  if (!kDelegate(cc).IsEmpty()) {
-    mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
-        kDelegate(cc).Get();
-    CHECK(input_side_packet_delegate.has_gpu() ||
-          input_side_packet_delegate.delegate_case() ==
-              mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
-        << "inference_calculator_gl_advanced only supports delegate input side "
-           "packet for Gpu";
-    delegate.MergeFrom(input_side_packet_delegate);
-  }
-  allow_precision_loss_ = delegate.gpu().allow_precision_loss();
-  tflite_gpu_runner_api_ = delegate.gpu().api();
-  tflite_gpu_runner_usage_ = delegate.gpu().usage();
-  use_kernel_caching_ = delegate.gpu().has_cached_kernel_path();
-  use_serialized_model_ = delegate.gpu().has_serialized_model_dir() &&
-                          delegate.gpu().has_model_token();
-
-  if (use_kernel_caching_) {
-#ifdef MEDIAPIPE_ANDROID
-    cached_kernel_filename_ = delegate.gpu().cached_kernel_path() +
-                              mediapipe::File::Basename(options.model_path()) +
-                              ".ker";
-#endif  // MEDIAPIPE_ANDROID
-  }
-  if (use_serialized_model_) {
-#ifdef MEDIAPIPE_ANDROID
-    serialized_model_path_ = mediapipe::file::JoinPath(
-        delegate.gpu().serialized_model_dir(), delegate.gpu().model_token());
-#endif  // MEDIAPIPE_ANDROID
-  }
-
+absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init(
+    CalculatorContext* cc,
+    const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
   MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
-  return gpu_helper_.RunInGlContext(
-      [this, &cc]() -> absl::Status { return InitTFLiteGPURunner(cc); });
+
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
+  MP_RETURN_IF_ERROR(on_disk_cache_helper_.Init(options, delegate.gpu()));
+
+  return gpu_helper_.RunInGlContext([this, &cc, &delegate]() -> absl::Status {
+    return InitTFLiteGPURunner(cc, delegate);
+  });
 }
 
-absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
-  if (kInTensors(cc).IsEmpty()) {
-    return absl::OkStatus();
-  }
-  const auto& input_tensors = *kInTensors(cc);
-  RET_CHECK(!input_tensors.empty());
-  auto output_tensors = absl::make_unique<std::vector<Tensor>>();
+absl::StatusOr<std::vector<Tensor>>
+InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
+    const std::vector<Tensor>& input_tensors) {
+  std::vector<Tensor> output_tensors;
 
   MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
       [this, &input_tensors, &output_tensors]() -> absl::Status {
@@ -142,90 +130,46 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
           MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
               input_tensors[i].GetOpenGlBufferReadView().name(), i));
         }
-        output_tensors->reserve(output_shapes_.size());
+        output_tensors.reserve(output_shapes_.size());
         for (int i = 0; i < output_shapes_.size(); ++i) {
-          output_tensors->emplace_back(Tensor::ElementType::kFloat32,
-                                       output_shapes_[i]);
+          output_tensors.emplace_back(Tensor::ElementType::kFloat32,
+                                      output_shapes_[i]);
           MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
-              output_tensors->back().GetOpenGlBufferWriteView().name(), i));
+              output_tensors.back().GetOpenGlBufferWriteView().name(), i));
         }
-        return absl::OkStatus();
+        // Run inference.
+        return tflite_gpu_runner_->Invoke();
       }));
 
-  // Run inference.
-  MP_RETURN_IF_ERROR(tflite_gpu_runner_->Invoke());
-  kOutTensors(cc).Send(std::move(output_tensors));
-  return absl::OkStatus();
+  return output_tensors;
 }
 
-absl::Status InferenceCalculatorGlAdvancedImpl::SaveGpuCaches() {
-#ifdef MEDIAPIPE_ANDROID
-  if (use_kernel_caching_) {
-    // Save kernel file.
-    auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
-        tflite_gpu_runner_->GetSerializedBinaryCache());
-    std::string cache_str(kernel_cache->begin(), kernel_cache->end());
-    MP_RETURN_IF_ERROR(
-        mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
-  }
-  if (use_serialized_model_) {
-    // Save serialized model file.
-    ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
-                     tflite_gpu_runner_->GetSerializedModel());
-    absl::string_view serialized_model(
-        reinterpret_cast<char*>(serialized_model_vec.data()),
-        serialized_model_vec.size());
-    MP_RETURN_IF_ERROR(
-        mediapipe::file::SetContents(serialized_model_path_, serialized_model));
-  }
-#endif  // MEDIAPIPE_ANDROID
-  return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
-  MP_RETURN_IF_ERROR(SaveGpuCaches());
+absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
+  MP_RETURN_IF_ERROR(
+      on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
   return gpu_helper_.RunInGlContext([this]() -> absl::Status {
     tflite_gpu_runner_.reset();
     return absl::OkStatus();
   });
 }
 
-absl::Status InferenceCalculatorGlAdvancedImpl::ReadGpuCaches() {
-#ifdef MEDIAPIPE_ANDROID
-  if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
-    // Load pre-compiled kernel file.
-    std::string cache_str;
-    MP_RETURN_IF_ERROR(
-        mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
-    std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
-    tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
-  }
-  if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
-    // Load serialized model file.
-    std::string serialized_model_str;
-    MP_RETURN_IF_ERROR(
-        file::GetContents(serialized_model_path_, &serialized_model_str));
-    std::vector<uint8_t> serialized_model_vec(serialized_model_str.begin(),
-                                              serialized_model_str.end());
-    tflite_gpu_runner_->SetSerializedModel(std::move(serialized_model_vec));
-  }
-#endif  // MEDIAPIPE_ANDROID
-  return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
-    CalculatorContext* cc) {
+absl::Status
+InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
+    CalculatorContext* cc,
+    const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
   ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
   const auto& model = *model_packet_.Get();
 
+  bool allow_precision_loss = delegate.gpu().allow_precision_loss();
+
   // Create runner
   tflite::gpu::InferenceOptions options;
-  options.priority1 = allow_precision_loss_
+  options.priority1 = allow_precision_loss
                           ? tflite::gpu::InferencePriority::MIN_LATENCY
                           : tflite::gpu::InferencePriority::MAX_PRECISION;
   options.priority2 = tflite::gpu::InferencePriority::AUTO;
   options.priority3 = tflite::gpu::InferencePriority::AUTO;
-  switch (tflite_gpu_runner_usage_) {
+  switch (delegate.gpu().usage()) {
     case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
         FAST_SINGLE_ANSWER: {
       options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
@@ -241,7 +185,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
     }
   }
   tflite_gpu_runner_ = std::make_unique<tflite::gpu::TFLiteGPURunner>(options);
-  switch (tflite_gpu_runner_api_) {
+  switch (delegate.gpu().api()) {
     case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
       // Do not need to force any specific API.
       break;
@@ -277,9 +221,148 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
                          tflite_gpu_runner_->GetOutputShapes()[i].c};
   }
 
-  MP_RETURN_IF_ERROR(ReadGpuCaches());
+  MP_RETURN_IF_ERROR(
+      on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
   return tflite_gpu_runner_->Build();
 }
 
+#if defined(MEDIAPIPE_ANDROID)
+absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
+    const mediapipe::InferenceCalculatorOptions& options,
+    const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+        gpu_delegate_options) {
+  use_kernel_caching_ = gpu_delegate_options.has_cached_kernel_path();
+  use_serialized_model_ = gpu_delegate_options.has_serialized_model_dir() &&
+                          gpu_delegate_options.has_model_token();
+
+  if (use_kernel_caching_) {
+    cached_kernel_filename_ = gpu_delegate_options.cached_kernel_path() +
+                              mediapipe::File::Basename(options.model_path()) +
+                              ".ker";
+  }
+  if (use_serialized_model_) {
+    serialized_model_path_ =
+        mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
+                                  gpu_delegate_options.model_token());
+  }
+  return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
+    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+  if (use_kernel_caching_) {
+    // Save kernel file.
+    auto kernel_cache = absl::make_unique<std::vector<uint8_t>>(
+        gpu_runner->GetSerializedBinaryCache());
+    std::string cache_str(kernel_cache->begin(), kernel_cache->end());
+    MP_RETURN_IF_ERROR(
+        mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
+  }
+  if (use_serialized_model_) {
+    // Save serialized model file.
+    ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
+                     gpu_runner->GetSerializedModel());
+    absl::string_view serialized_model(
+        reinterpret_cast<char*>(serialized_model_vec.data()),
+        serialized_model_vec.size());
+    MP_RETURN_IF_ERROR(
+        mediapipe::file::SetContents(serialized_model_path_, serialized_model));
+  }
+  return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
+    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+  if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
+    // Load pre-compiled kernel file.
+    std::string cache_str;
+    MP_RETURN_IF_ERROR(
+        mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
+    std::vector<uint8_t> cache_vec(cache_str.begin(), cache_str.end());
+    gpu_runner->SetSerializedBinaryCache(std::move(cache_vec));
+  }
+  if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
+    // Load serialized model file.
+    std::string serialized_model_str;
+    MP_RETURN_IF_ERROR(
+        file::GetContents(serialized_model_path_, &serialized_model_str));
+    std::vector<uint8_t> serialized_model_vec(serialized_model_str.begin(),
+                                              serialized_model_str.end());
+    gpu_runner->SetSerializedModel(std::move(serialized_model_vec));
+  }
+  return absl::OkStatus();
+}
+#else
+absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
+    const mediapipe::InferenceCalculatorOptions& options,
+    const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+        gpu_delegate_options) {
+  return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
+    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+  return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
+    tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+  return absl::OkStatus();
+}
+#endif  // MEDIAPIPE_ANDROID
+
+absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
+    CalculatorContract* cc) {
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
+  RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
+      << "Either model as side packet or model path in options is required.";
+
+  MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
+  mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
+  if (!kDelegate(cc).IsEmpty()) {
+    const mediapipe::InferenceCalculatorOptions::Delegate&
+        input_side_packet_delegate = kDelegate(cc).Get();
+    RET_CHECK(
+        input_side_packet_delegate.has_gpu() ||
+        input_side_packet_delegate.delegate_case() ==
+            mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+        << "inference_calculator_gl_advanced only supports gpu delegate "
+           "configuration through side packet.";
+    delegate.MergeFrom(input_side_packet_delegate);
+  }
+
+  gpu_inference_runner_ = std::make_unique<GpuInferenceRunner>();
+  return gpu_inference_runner_->Init(cc, delegate);
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
+  if (kInTensors(cc).IsEmpty()) {
+    return absl::OkStatus();
+  }
+
+  const auto& input_tensors = *kInTensors(cc);
+  RET_CHECK(!input_tensors.empty());
+  auto output_tensors = absl::make_unique<std::vector<Tensor>>();
+
+  ASSIGN_OR_RETURN(*output_tensors,
+                   gpu_inference_runner_->Process(input_tensors));
+
+  kOutTensors(cc).Send(std::move(output_tensors));
+  return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
+  return gpu_inference_runner_->Close();
+}
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/tensor/inference_calculator_metal.cc

@@ -116,7 +116,9 @@ class InferenceCalculatorMetalImpl
 
 absl::Status InferenceCalculatorMetalImpl::UpdateContract(
     CalculatorContract* cc) {
-  const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
+  RET_CHECK(!kDelegate(cc).IsConnected())
+      << "Delegate configuration through side packet is not supported.";
+  const auto& options = cc->Options<mediapipe::InferenceCalculatorOptions>();
   RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
       << "Either model as side packet or model path in options is required.";
 

mediapipe/calculators/tensor/inference_calculator_test.cc

@@ -16,13 +16,17 @@
 #include <string>
 #include <vector>
 
+#include "absl/log/check.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/string_view.h"
 #include "mediapipe/calculators/tensor/inference_calculator.pb.h"
+#include "mediapipe/calculators/tensor/inference_calculator_test_base.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/deps/file_path.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/benchmark.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -118,9 +122,11 @@ void RunGraphThenClose(CalculatorGraph& graph, std::vector<Tensor> input_vec) {
   MP_ASSERT_OK(graph.StartRun({}));
 
   // Push the tensor into the graph.
-  MP_ASSERT_OK(graph.AddPacketToInputStream(
-      "tensor_in",
-      MakePacket<std::vector<Tensor>>(std::move(input_vec)).At(Timestamp(0))));
+  if (!input_vec.empty()) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "tensor_in", MakePacket<std::vector<Tensor>>(std::move(input_vec))
+                         .At(Timestamp(0))));
+  }
   // Wait until the calculator done processing.
   MP_ASSERT_OK(graph.WaitUntilIdle());
 
@@ -174,5 +180,13 @@ TEST(InferenceCalculatorTest, ModelAsInputSidePacketSmokeTest) {
   DoSmokeTest(kGraphWithModelAsInputSidePacket);
 }
 
+void BM_InitializeCalculator(benchmark::State& state) {
+  mediapipe::InferenceCalculatorOptions::Delegate delegate;
+  delegate.mutable_tflite();
+  RunBenchmarkCalculatorInitialization(state, delegate);
+}
+
+BENCHMARK(BM_InitializeCalculator);
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc

@@ -15,6 +15,8 @@
 #include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
 
 #include <memory>
+#include <optional>
+#include <type_traits>
 
 #include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
@@ -28,8 +30,25 @@ namespace api2 {
 
 namespace {
 
+// Returns the scale attribute should be multiplied by.
+float GetAttributeScale(
+    const LandmarksToTensorCalculatorOptions::Attribute& attribute,
+    const std::pair<int, int>& image_size) {
+  switch (attribute) {
+    case LandmarksToTensorCalculatorOptions::X:
+    case LandmarksToTensorCalculatorOptions::Z:
+      return image_size.first;
+    case LandmarksToTensorCalculatorOptions::Y:
+      return image_size.second;
+    case LandmarksToTensorCalculatorOptions::VISIBILITY:
+    case LandmarksToTensorCalculatorOptions::PRESENCE:
+      return 1.0f;
+  }
+}
+
+template <typename LandmarkType>
 float GetAttribute(
-    const Landmark& landmark,
+    const LandmarkType& landmark,
     const LandmarksToTensorCalculatorOptions::Attribute& attribute) {
   switch (attribute) {
     case LandmarksToTensorCalculatorOptions::X:
@@ -45,6 +64,33 @@ float GetAttribute(
   }
 }
 
+template <typename LandmarksT>
+Tensor ConvertLandmarksToTensor(
+    const LandmarksT& landmarks, const std::vector<float>& attribute_scales,
+    const LandmarksToTensorCalculatorOptions& options) {
+  // Determine tensor shape.
+  const int n_landmarks = landmarks.landmark_size();
+  const int n_attributes = options.attributes_size();
+  auto tensor_shape = options.flatten()
+                          ? Tensor::Shape{1, n_landmarks * n_attributes}
+                          : Tensor::Shape{1, n_landmarks, n_attributes};
+
+  // Create empty tesnor.
+  Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
+  auto* buffer = tensor.GetCpuWriteView().buffer<float>();
+
+  // Fill tensor with landmark attributes.
+  for (int i = 0; i < n_landmarks; ++i) {
+    for (int j = 0; j < n_attributes; ++j) {
+      float value = GetAttribute(landmarks.landmark(i), options.attributes(j));
+      float scale = attribute_scales[j];
+      buffer[i * n_attributes + j] = value * scale;
+    }
+  }
+
+  return tensor;
+}
+
 }  // namespace
 
 class LandmarksToTensorCalculatorImpl
@@ -54,39 +100,52 @@ class LandmarksToTensorCalculatorImpl
     options_ = cc->Options<LandmarksToTensorCalculatorOptions>();
     RET_CHECK(options_.attributes_size() > 0)
         << "At least one attribute must be specified";
+
+    RET_CHECK(kInLandmarkList(cc).IsConnected() ^
+              kInNormLandmarkList(cc).IsConnected())
+        << "Exactly one landmarks input should be provided";
+    RET_CHECK_EQ(kInNormLandmarkList(cc).IsConnected(),
+                 kImageSize(cc).IsConnected())
+        << "Image size should be provided only for normalized landmarks";
+
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) override {
-    if (kInLandmarkList(cc).IsEmpty()) {
-      return absl::OkStatus();
-    }
-
-    // Get input landmarks.
-    const auto& in_landmarks = *kInLandmarkList(cc);
-
-    // Determine tensor shape.
-    const int n_landmarks = in_landmarks.landmark_size();
-    const int n_attributes = options_.attributes_size();
-    auto tensor_shape = options_.flatten()
-                            ? Tensor::Shape{1, n_landmarks * n_attributes}
-                            : Tensor::Shape{1, n_landmarks, n_attributes};
-
-    // Create empty tesnor.
-    Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
-    auto* buffer = tensor.GetCpuWriteView().buffer<float>();
-
-    // Fill tensor with landmark attributes.
-    for (int i = 0; i < n_landmarks; ++i) {
-      for (int j = 0; j < n_attributes; ++j) {
-        buffer[i * n_attributes + j] =
-            GetAttribute(in_landmarks.landmark(i), options_.attributes(j));
+    // Get attribute scales depending on whether landmarks are normalized or
+    // not.
+    std::vector<float> attribute_scales;
+    if (kInLandmarkList(cc).IsConnected()) {
+      for (int j = 0; j < options_.attributes_size(); ++j) {
+        attribute_scales.push_back(1.0f);
+      }
+    } else {
+      RET_CHECK(!kImageSize(cc).IsEmpty());
+      auto image_size = kImageSize(cc).Get();
+      for (int j = 0; j < options_.attributes_size(); ++j) {
+        attribute_scales.push_back(
+            GetAttributeScale(options_.attributes(j), image_size));
       }
     }
 
-    // Return vector with a single tensor.
+    // Convert landmarks to tensor.
     auto result = std::vector<Tensor>();
-    result.push_back(std::move(tensor));
+    if (kInLandmarkList(cc).IsConnected()) {
+      if (kInLandmarkList(cc).IsEmpty()) {
+        return absl::OkStatus();
+      }
+      Tensor tensor = ConvertLandmarksToTensor(kInLandmarkList(cc).Get(),
+                                               attribute_scales, options_);
+      result.push_back(std::move(tensor));
+    } else {
+      if (kInNormLandmarkList(cc).IsEmpty()) {
+        return absl::OkStatus();
+      }
+      Tensor tensor = ConvertLandmarksToTensor(kInNormLandmarkList(cc).Get(),
+                                               attribute_scales, options_);
+      result.push_back(std::move(tensor));
+    }
+
     kOutTensors(cc).Send(std::move(result));
 
     return absl::OkStatus();

mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h

@@ -28,8 +28,12 @@ namespace api2 {
 // A calculator for converting landmars into a Tensor.
 //
 // Input:
-//   LANDMARKS - LandmarkList
+//   LANDMARKS (optional) - LandmarkList
 //     Landmarks to be converted into a Tensor.
+//   NORM_LANDMARKS (optional) - NormalizedLandmarkList.
+//     Normalized landmarks to be converted into a Tensor.
+//   IMAGE_SIZE (optional) - std::pair<int, int>
+//     Image size to scale NORM_LANDMARKS.
 //
 // Output:
 //   TENSORS - std::vector<Tensor>
@@ -49,10 +53,15 @@ namespace api2 {
 // }
 class LandmarksToTensorCalculator : public NodeIntf {
  public:
-  static constexpr Input<LandmarkList>::Optional kInLandmarkList{"LANDMARKS"};
+  static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarkList{
+      "LANDMARKS"};
+  static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
+      kInNormLandmarkList{"NORM_LANDMARKS"};
+  static constexpr Input<std::pair<int, int>>::Optional kImageSize{
+      "IMAGE_SIZE"};
   static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
   MEDIAPIPE_NODE_INTERFACE(LandmarksToTensorCalculator, kInLandmarkList,
-                           kOutTensors);
+                           kInNormLandmarkList, kImageSize, kOutTensors);
 };
 
 }  // namespace api2

mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc

@@ -40,6 +40,20 @@ void RunLandmarks(mediapipe::CalculatorRunner* runner,
   MP_ASSERT_OK(runner->Run());
 }
 
+void RunNormLandmarks(mediapipe::CalculatorRunner* runner,
+                      const NormalizedLandmarkList& landmarks,
+                      const std::pair<int, int> image_size) {
+  runner->MutableInputs()
+      ->Tag("NORM_LANDMARKS")
+      .packets.push_back(
+          MakePacket<NormalizedLandmarkList>(landmarks).At(Timestamp(0)));
+  runner->MutableInputs()
+      ->Tag("IMAGE_SIZE")
+      .packets.push_back(
+          MakePacket<std::pair<int, int>>(image_size).At(Timestamp(0)));
+  MP_ASSERT_OK(runner->Run());
+}
+
 const Tensor& GetOutputTensor(mediapipe::CalculatorRunner* runner) {
   const auto& output_packets = runner->Outputs().Tag("TENSORS").packets;
   EXPECT_EQ(output_packets.size(), 1);
@@ -151,5 +165,34 @@ TEST(LandmarksToTensorCalculatorTest, XYZAttributes_Flatten) {
                  {1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
 }
 
+TEST(LandmarksToTensorCalculatorTest, NormalizedLandmarks) {
+  mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"pb(
+    calculator: "LandmarksToTensorCalculator"
+    input_stream: "NORM_LANDMARKS:landmarks"
+    input_stream: "IMAGE_SIZE:image_size"
+    output_stream: "TENSORS:tensors"
+    options: {
+      [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+        attributes: [ X, Y, Z, VISIBILITY, PRESENCE ]
+      }
+    }
+  )pb"));
+
+  NormalizedLandmarkList landmarks;
+  auto* landmark1 = landmarks.add_landmark();
+  landmark1->set_x(0.1f);
+  landmark1->set_y(0.5f);
+  landmark1->set_z(1.0f);
+  landmark1->set_visibility(4.0f);
+  landmark1->set_presence(5.0f);
+
+  std::pair<int, int> image_size{200, 100};
+
+  RunNormLandmarks(&runner, landmarks, image_size);
+  const auto& tensor = GetOutputTensor(&runner);
+  ValidateTensor(tensor, /*expected_shape=*/{1, 1, 5}, /*expected_values=*/
+                 {20.0f, 50.0f, 200.0f, 4.0f, 5.0f});
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/tensor/tensors_dequantization_calculator.cc

@@ -0,0 +1,102 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+namespace {
+
+template <typename T>
+void Dequantize(const Tensor& input, Tensor* output) {
+  auto input_view = input.GetCpuReadView();
+  auto input_buffer = input_view.buffer<T>();
+  auto output_view = output->GetCpuWriteView();
+  auto output_buffer = output_view.buffer<float>();
+  for (int i = 0; i < input.shape().num_elements(); ++i) {
+    output_buffer[i] = input.quantization_parameters().scale *
+                       (static_cast<int>(input_buffer[i]) -
+                        input.quantization_parameters().zero_point);
+  }
+}
+
+}  // namespace
+
+// Performs dequantization using the quantization parameters from the input
+// UInt8 or Int8 tensors. Each element of the input tensors is converted using:
+//
+//   output = quantization_parameters.scale *
+//     (input - quantization_parameters.zero_point)
+//
+// Input:
+//  TENSORS - Vector of quantized Tensors of type kUint8 or kInt8.
+// Output:
+//  TENSORS - Vector of dequantized Tensors of type kFloat32.
+//
+// Usage example:
+// node {
+//   calculator: "TensorsDequantizationCalculator"
+//   input_stream: "TENSORS:quantized_tensors"
+//   output_stream: "TENSORS:dequantized_tensors"
+// }
+class TensorsDequantizationCalculator : public Node {
+ public:
+  static constexpr Input<std::vector<Tensor>> kInTensors{"TENSORS"};
+  static constexpr Output<std::vector<Tensor>> kOutTensors{"TENSORS"};
+  MEDIAPIPE_NODE_CONTRACT(kInTensors, kOutTensors);
+
+  absl::Status Process(CalculatorContext* cc) override;
+};
+
+absl::Status TensorsDequantizationCalculator::Process(CalculatorContext* cc) {
+  if (kInTensors(cc).IsEmpty()) {
+    return absl::OkStatus();
+  }
+  const auto& input_tensors = *kInTensors(cc);
+  RET_CHECK(!input_tensors.empty());
+  auto output_tensors = std::make_unique<std::vector<Tensor>>();
+  output_tensors->reserve(input_tensors.size());
+  for (const auto& input_tensor : input_tensors) {
+    output_tensors->emplace_back(Tensor::ElementType::kFloat32,
+                                 input_tensor.shape());
+    switch (input_tensor.element_type()) {
+      case Tensor::ElementType::kUInt8:
+        Dequantize<uint8>(input_tensor, &output_tensors->back());
+        break;
+      case Tensor::ElementType::kInt8:
+        Dequantize<int8>(input_tensor, &output_tensors->back());
+        break;
+      default:
+        return absl::InvalidArgumentError(absl::StrCat(
+            "Unsupported input tensor type: ", input_tensor.element_type()));
+    }
+  }
+  kOutTensors(cc).Send(std::move(output_tensors));
+  return absl::OkStatus();
+}
+
+MEDIAPIPE_REGISTER_NODE(TensorsDequantizationCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc

@@ -0,0 +1,128 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::ParseTextProtoOrDie;
+using ::testing::HasSubstr;
+using Node = ::mediapipe::CalculatorGraphConfig::Node;
+
+constexpr char kCalculatorConfig[] = R"pb(
+  calculator: "TensorsDequantizationCalculator"
+  input_stream: "TENSORS:input"
+  output_stream: "TENSORS:output"
+)pb";
+
+// Compares the provided tensor contents with the expected values.
+void ValidateResult(const Tensor& actual, const std::vector<float>& expected) {
+  EXPECT_EQ(actual.element_type(), Tensor::ElementType::kFloat32);
+  EXPECT_EQ(expected.size(), actual.shape().num_elements());
+  auto view = actual.GetCpuReadView();
+  auto buffer = view.buffer<float>();
+  for (int i = 0; i < expected.size(); ++i) {
+    EXPECT_FLOAT_EQ(expected[i], buffer[i]);
+  }
+}
+
+class TensorsDequantizationCalculatorTest : public ::testing::Test {
+ protected:
+  TensorsDequantizationCalculatorTest()
+      : runner_(ParseTextProtoOrDie<Node>(kCalculatorConfig)) {}
+
+  template <typename T>
+  void PushTensor(Tensor::ElementType type, std::vector<T> tensor,
+                  std::optional<Tensor::QuantizationParameters>
+                      quantization_params = std::nullopt) {
+    auto tensors = std::make_unique<std::vector<Tensor>>();
+    if (quantization_params.has_value()) {
+      tensors->emplace_back(type,
+                            Tensor::Shape{static_cast<int>(tensor.size())},
+                            quantization_params.value());
+    } else {
+      tensors->emplace_back(type,
+                            Tensor::Shape{static_cast<int>(tensor.size())});
+    }
+    auto view = tensors->back().GetCpuWriteView();
+    auto buffer = view.buffer<T>();
+    std::copy(tensor.begin(), tensor.end(), buffer);
+    runner_.MutableInputs()->Tag("TENSORS").packets.push_back(
+        Adopt(tensors.release()).At(Timestamp(0)));
+  }
+
+  const Tensor& GetOutput() {
+    return runner_.Outputs()
+        .Get("TENSORS", 0)
+        .packets[0]
+        .Get<std::vector<Tensor>>()[0];
+  }
+
+  CalculatorRunner runner_;
+};
+
+TEST_F(TensorsDequantizationCalculatorTest, FailsWithFloatTensors) {
+  std::vector<float> tensor = {0, 1};
+  PushTensor(Tensor::ElementType::kFloat32, tensor);
+
+  auto status = runner_.Run();
+
+  EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
+  EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, FailsWithInt32Tensors) {
+  std::vector<int32_t> tensor = {0, 1};
+  PushTensor(Tensor::ElementType::kInt32, tensor);
+
+  auto status = runner_.Run();
+
+  EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
+  EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithUInt8Tensors) {
+  std::vector<uint8_t> tensor = {0, 127, 255};
+  PushTensor(Tensor::ElementType::kUInt8, tensor,
+             Tensor::QuantizationParameters{1.0f / 127, 127});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  ValidateResult(GetOutput(), {-1, 0, 1.007874});
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithInt8Tensors) {
+  std::vector<int8_t> tensor = {-128, 0, 127};
+  PushTensor(Tensor::ElementType::kInt8, tensor,
+             Tensor::QuantizationParameters{1.0f / 127, 0});
+
+  MP_ASSERT_OK(runner_.Run());
+
+  ValidateResult(GetOutput(), {-1.007874, 0, 1});
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_classification_calculator.cc

@@ -165,6 +165,7 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToClassificationCalculator::Process(CalculatorContext* cc) {
   const auto& input_tensors = *kInTensors(cc);
   RET_CHECK_EQ(input_tensors.size(), 1);
+  RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
 
   int num_classes = input_tensors[0].shape().num_elements();
 

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -287,7 +287,11 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
       }
     }
   }
-  const int num_input_tensors = kInTensors(cc)->size();
+  const auto& input_tensors = *kInTensors(cc);
+  for (const auto& tensor : input_tensors) {
+    RET_CHECK(tensor.element_type() == Tensor::ElementType::kFloat32);
+  }
+  const int num_input_tensors = input_tensors.size();
   if (!scores_tensor_index_is_set_) {
     if (num_input_tensors == 2 ||
         num_input_tensors == kNumInputTensorsWithAnchors) {

mediapipe/calculators/tensor/tensors_to_floats_calculator.cc

@@ -76,6 +76,7 @@ absl::Status TensorsToFloatsCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToFloatsCalculator::Process(CalculatorContext* cc) {
   const auto& input_tensors = *kInTensors(cc);
   RET_CHECK(!input_tensors.empty());
+  RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
   // TODO: Add option to specify which tensor to take from.
   auto view = input_tensors[0].GetCpuReadView();
   auto raw_floats = view.buffer<float>();

mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc

@@ -139,6 +139,7 @@ absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) {
   bool flip_vertically = kFlipVertically(cc).GetOr(options_.flip_vertically());
 
   const auto& input_tensors = *kInTensors(cc);
+  RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
   int num_values = input_tensors[0].shape().num_elements();
   const int num_dimensions = num_values / num_landmarks_;
   CHECK_GT(num_dimensions, 0);

mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc

@@ -116,8 +116,9 @@ using ::tflite::gpu::gl::GlShader;
 //
 // Inputs:
 //   One of the following TENSORS tags:
-//   TENSORS: Vector of Tensor,
-//            The tensor dimensions are specified in this calculator's options.
+//   TENSORS: Vector of Tensors of type kFloat32. Only the first tensor will be
+//            used. The tensor dimensions are specified in this calculator's
+//            options.
 //   OUTPUT_SIZE(optional): std::pair<int, int>,
 //                          If provided, the size to upscale mask to.
 //
@@ -261,6 +262,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
   // Validate tensor channels and activation type.
   {
     RET_CHECK(!input_tensors.empty());
+    RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
     ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
     int tensor_channels = std::get<2>(hwc);
     typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;

mediapipe/calculators/tensorflow/BUILD

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
 
 licenses(["notice"])
 
@@ -88,6 +88,13 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "tensor_to_vector_int_calculator_options_proto",
+    srcs = ["tensor_to_vector_int_calculator_options.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 proto_library(
     name = "tensor_to_vector_string_calculator_options_proto",
     srcs = ["tensor_to_vector_string_calculator_options.proto"],
@@ -95,10 +102,12 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "unpack_media_sequence_calculator_proto",
     srcs = ["unpack_media_sequence_calculator.proto"],
-    visibility = ["//visibility:public"],
+    visibility = [
+        "//visibility:public",
+    ],
     deps = [
         "//mediapipe/calculators/core:packet_resampler_calculator_proto",
         "//mediapipe/framework:calculator_proto",
@@ -166,17 +175,6 @@ mediapipe_cc_proto_library(
     deps = [":object_detection_tensors_to_detections_calculator_proto"],
 )
 
-mediapipe_cc_proto_library(
-    name = "pack_media_sequence_calculator_cc_proto",
-    srcs = ["pack_media_sequence_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":pack_media_sequence_calculator_proto"],
-)
-
 mediapipe_cc_proto_library(
     name = "tensorflow_inference_calculator_cc_proto",
     srcs = ["tensorflow_inference_calculator.proto"],
@@ -264,6 +262,14 @@ mediapipe_cc_proto_library(
     deps = [":tensor_to_vector_float_calculator_options_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tensor_to_vector_int_calculator_options_cc_proto",
+    srcs = ["tensor_to_vector_int_calculator_options.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":tensor_to_vector_int_calculator_options_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "tensor_to_vector_string_calculator_options_cc_proto",
     srcs = ["tensor_to_vector_string_calculator_options.proto"],
@@ -272,18 +278,6 @@ mediapipe_cc_proto_library(
     deps = [":tensor_to_vector_string_calculator_options_proto"],
 )
 
-mediapipe_cc_proto_library(
-    name = "unpack_media_sequence_calculator_cc_proto",
-    srcs = ["unpack_media_sequence_calculator.proto"],
-    cc_deps = [
-        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
-        "//mediapipe/framework:calculator_cc_proto",
-        "//mediapipe/util:audio_decoder_cc_proto",
-    ],
-    visibility = ["//visibility:public"],
-    deps = [":unpack_media_sequence_calculator_proto"],
-)
-
 mediapipe_cc_proto_library(
     name = "vector_int_to_tensor_calculator_options_cc_proto",
     srcs = ["vector_int_to_tensor_calculator_options.proto"],
@@ -420,8 +414,9 @@ cc_library(
         "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto",
         "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:detection_cc_proto",  # build_cleaner: keep
         "//mediapipe/framework/formats:location",
+        "//mediapipe/framework/formats:location_opencv",
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -458,6 +453,7 @@ cc_library(
     deps = [
         ":tensorflow_session",
         ":tensorflow_inference_calculator_cc_proto",
+        "@com_google_absl//absl/log:check",
         "//mediapipe/framework:timestamp",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/memory",
@@ -722,6 +718,28 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tensor_to_vector_int_calculator",
+    srcs = ["tensor_to_vector_int_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tensor_to_vector_int_calculator_options_cc_proto",
+        "@com_google_absl//absl/base:core_headers",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:ret_check",
+    ] + select({
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/core:framework",
+        ],
+        "//mediapipe:android": [
+            "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tensor_to_vector_string_calculator",
     srcs = ["tensor_to_vector_string_calculator.cc"],
@@ -916,6 +934,7 @@ cc_test(
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:location",
+        "//mediapipe/framework/formats:location_opencv",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/util/sequence:media_sequence",
@@ -1106,6 +1125,20 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "tensor_to_vector_int_calculator_test",
+    srcs = ["tensor_to_vector_int_calculator_test.cc"],
+    deps = [
+        ":tensor_to_vector_int_calculator",
+        ":tensor_to_vector_int_calculator_options_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/port:gtest_main",
+        "@org_tensorflow//tensorflow/core:framework",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
+    ],
+)
+
 cc_test(
     name = "tensor_to_vector_string_calculator_test",
     srcs = ["tensor_to_vector_string_calculator_test.cc"],

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -22,6 +22,7 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/formats/location_opencv.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -37,6 +38,7 @@ const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 const char kImageTag[] = "IMAGE";
 const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
 const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
+const char kIntFeaturePrefixTag[] = "INT_FEATURE_";
 const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_";
 const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
 const char kBBoxTag[] = "BBOX";
@@ -88,7 +90,7 @@ namespace mpms = mediapipe::mediasequence;
 // }
 namespace {
 uint8 ConvertFloatToByte(const float float_value) {
-  float clamped_value = MathUtil::Clamp(0.0f, 1.0f, float_value);
+  float clamped_value = std::clamp(0.0f, 1.0f, float_value);
   return static_cast<uint8>(clamped_value * 255.0 + .5f);
 }
 }  // namespace
@@ -154,6 +156,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
       if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
         cc->Inputs().Tag(tag).Set<std::vector<float>>();
       }
+      if (absl::StartsWith(tag, kIntFeaturePrefixTag)) {
+        cc->Inputs().Tag(tag).Set<std::vector<int64>>();
+      }
       if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
         cc->Inputs().Tag(tag).Set<std::vector<std::string>>();
       }
@@ -235,6 +240,12 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           mpms::ClearFeatureFloats(key, sequence_.get());
           mpms::ClearFeatureTimestamp(key, sequence_.get());
         }
+        if (absl::StartsWith(tag, kIntFeaturePrefixTag)) {
+          std::string key = tag.substr(
+              sizeof(kIntFeaturePrefixTag) / sizeof(*kIntFeaturePrefixTag) - 1);
+          mpms::ClearFeatureInts(key, sequence_.get());
+          mpms::ClearFeatureTimestamp(key, sequence_.get());
+        }
         if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
           std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
                                            sizeof(*kBytesFeaturePrefixTag) -
@@ -416,6 +427,16 @@ class PackMediaSequenceCalculator : public CalculatorBase {
                                cc->Inputs().Tag(tag).Get<std::vector<float>>(),
                                sequence_.get());
       }
+      if (absl::StartsWith(tag, kIntFeaturePrefixTag) &&
+          !cc->Inputs().Tag(tag).IsEmpty()) {
+        std::string key = tag.substr(
+            sizeof(kIntFeaturePrefixTag) / sizeof(*kIntFeaturePrefixTag) - 1);
+        mpms::AddFeatureTimestamp(key, cc->InputTimestamp().Value(),
+                                  sequence_.get());
+        mpms::AddFeatureInts(key,
+                             cc->Inputs().Tag(tag).Get<std::vector<int64>>(),
+                             sequence_.get());
+      }
       if (absl::StartsWith(tag, kBytesFeaturePrefixTag) &&
           !cc->Inputs().Tag(tag).IsEmpty()) {
         std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
@@ -508,7 +529,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
           RET_CHECK(!already_has_mask)
               << "We currently only support adding one mask per timestamp. "
               << sequence_->DebugString();
-          auto mask_mat_ptr = Location(detection.location_data()).GetCvMask();
+          auto mask_mat_ptr = GetCvMask(Location(detection.location_data()));
           std::vector<uchar> bytes;
           RET_CHECK(cv::imencode(".png", *mask_mat_ptr, bytes, {}));
 

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -25,6 +25,7 @@
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/formats/location_opencv.h"
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
@@ -56,6 +57,8 @@ constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
 constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
 constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
 constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
+constexpr char kIntFeatureOtherTag[] = "INT_FEATURE_OTHER";
+constexpr char kIntFeatureTestTag[] = "INT_FEATURE_TEST";
 constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
 constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
 constexpr char kImageTag[] = "IMAGE";
@@ -217,6 +220,50 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
   }
 }
 
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoIntLists) {
+  SetUpCalculator({"INT_FEATURE_TEST:test", "INT_FEATURE_OTHER:test2"}, {},
+                  false, true);
+  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
+
+  int num_timesteps = 2;
+  for (int i = 0; i < num_timesteps; ++i) {
+    auto vi_ptr = ::absl::make_unique<std::vector<int64>>(2, 2 << i);
+    runner_->MutableInputs()
+        ->Tag(kIntFeatureTestTag)
+        .packets.push_back(Adopt(vi_ptr.release()).At(Timestamp(i)));
+    vi_ptr = ::absl::make_unique<std::vector<int64>>(2, 2 << i);
+    runner_->MutableInputs()
+        ->Tag(kIntFeatureOtherTag)
+        .packets.push_back(Adopt(vi_ptr.release()).At(Timestamp(i)));
+  }
+
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Tag(kSequenceExampleTag).packets;
+  ASSERT_EQ(1, output_packets.size());
+  const tf::SequenceExample& output_sequence =
+      output_packets[0].Get<tf::SequenceExample>();
+
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("TEST", output_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureIntsSize("TEST", output_sequence));
+  ASSERT_EQ(num_timesteps,
+            mpms::GetFeatureTimestampSize("OTHER", output_sequence));
+  ASSERT_EQ(num_timesteps, mpms::GetFeatureIntsSize("OTHER", output_sequence));
+  for (int i = 0; i < num_timesteps; ++i) {
+    ASSERT_EQ(i, mpms::GetFeatureTimestampAt("TEST", output_sequence, i));
+    ASSERT_THAT(mpms::GetFeatureIntsAt("TEST", output_sequence, i),
+                ::testing::ElementsAreArray(std::vector<int64>(2, 2 << i)));
+    ASSERT_EQ(i, mpms::GetFeatureTimestampAt("OTHER", output_sequence, i));
+    ASSERT_THAT(mpms::GetFeatureIntsAt("OTHER", output_sequence, i),
+                ::testing::ElementsAreArray(std::vector<int64>(2, 2 << i)));
+  }
+}
+
 TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) {
   SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
                   false, true);
@@ -434,7 +481,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
     detection.add_label("mask");
     detection.add_score(1.0);
     cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
-    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+    mediapipe::CreateCvMaskLocation<uint8>(image).ConvertToProto(
         detection.mutable_location_data());
     detections->push_back(detection);
 
@@ -513,7 +560,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) {
     detection.add_label("mask");
     detection.add_score(1.0);
     cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
-    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+    mediapipe::CreateCvMaskLocation<uint8>(image).ConvertToProto(
         detection.mutable_location_data());
     detections->push_back(detection);
 
@@ -561,7 +608,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
     detection.add_label("mask");
     detection.add_score(1.0);
     cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
-    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+    mediapipe::CreateCvMaskLocation<uint8>(image).ConvertToProto(
         detection.mutable_location_data());
     detections->push_back(detection);
 
@@ -677,7 +724,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
     detection.add_label("mask");
     detection.add_score(1.0);
     cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
-    Location::CreateCvMaskLocation<uint8>(image).ConvertToProto(
+    mediapipe::CreateCvMaskLocation<uint8>(image).ConvertToProto(
         detection.mutable_location_data());
 
     detections->push_back(detection);

mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc

@@ -0,0 +1,151 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
+// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
+
+#include <memory>
+
+#include "absl/base/integral_types.h"
+#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+
+namespace mediapipe {
+
+namespace tf = ::tensorflow;
+
+class TensorToVectorIntCalculator : public CalculatorBase {
+ public:
+  static absl::Status GetContract(CalculatorContract* cc);
+
+  absl::Status Open(CalculatorContext* cc) override;
+  absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+  void TokenizeVector(std::vector<int64>* vector) const;
+
+  TensorToVectorIntCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(TensorToVectorIntCalculator);
+
+absl::Status TensorToVectorIntCalculator::GetContract(CalculatorContract* cc) {
+  // Start with only one input packet.
+  RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
+      << "Only one input stream is supported.";
+  cc->Inputs().Index(0).Set<tf::Tensor>(
+      // Input Tensor
+  );
+  RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
+      << "Only one output stream is supported.";
+  const auto& options = cc->Options<TensorToVectorIntCalculatorOptions>();
+  if (options.tensor_is_2d()) {
+    RET_CHECK(!options.flatten_nd());
+    cc->Outputs().Index(0).Set<std::vector<std::vector<int64>>>(
+        /* "Output vector<vector<float>>." */);
+  } else {
+    cc->Outputs().Index(0).Set<std::vector<int64>>(
+        // Output vector<float>.
+    );
+  }
+  return absl::OkStatus();
+}
+
+absl::Status TensorToVectorIntCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<TensorToVectorIntCalculatorOptions>();
+
+  // Inform mediapipe that this calculator produces an output at time t for
+  // each input received at time t (i.e. this calculator does not buffer
+  // inputs). This enables mediapipe to propagate time of arrival estimates in
+  // mediapipe graphs through this calculator.
+  cc->SetOffset(/*offset=*/0);
+
+  return absl::OkStatus();
+}
+
+absl::Status TensorToVectorIntCalculator::Process(CalculatorContext* cc) {
+  const tf::Tensor& input_tensor =
+      cc->Inputs().Index(0).Value().Get<tf::Tensor>();
+  RET_CHECK(tf::DT_INT32 == input_tensor.dtype() ||
+            tf::DT_INT64 == input_tensor.dtype())
+      << "expected DT_INT32 or DT_INT64 input but got "
+      << tensorflow::DataTypeString(input_tensor.dtype());
+
+  if (options_.tensor_is_2d()) {
+    RET_CHECK(2 == input_tensor.dims())
+        << "Expected 2-dimensional Tensor, but the tensor shape is: "
+        << input_tensor.shape().DebugString();
+    auto output = absl::make_unique<std::vector<std::vector<int64>>>(
+        input_tensor.dim_size(0), std::vector<int64>(input_tensor.dim_size(1)));
+    for (int i = 0; i < input_tensor.dim_size(0); ++i) {
+      auto& instance_output = output->at(i);
+      if (tf::DT_INT32 == input_tensor.dtype()) {
+        const auto& slice =
+            input_tensor.Slice(i, i + 1).unaligned_flat<int32>();
+        for (int j = 0; j < input_tensor.dim_size(1); ++j) {
+          instance_output.at(j) = slice(j);
+        }
+      } else {
+        const auto& slice =
+            input_tensor.Slice(i, i + 1).unaligned_flat<int64>();
+        for (int j = 0; j < input_tensor.dim_size(1); ++j) {
+          instance_output.at(j) = slice(j);
+        }
+      }
+      TokenizeVector(&instance_output);
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  } else {
+    if (!options_.flatten_nd()) {
+      RET_CHECK(1 == input_tensor.dims())
+          << "`flatten_nd` is not set. Expected 1-dimensional Tensor, but the "
+          << "tensor shape is: " << input_tensor.shape().DebugString();
+    }
+    auto output =
+        absl::make_unique<std::vector<int64>>(input_tensor.NumElements());
+    if (tf::DT_INT32 == input_tensor.dtype()) {
+      const auto& tensor_values = input_tensor.flat<int32>();
+      for (int i = 0; i < input_tensor.NumElements(); ++i) {
+        output->at(i) = tensor_values(i);
+      }
+    } else {
+      const auto& tensor_values = input_tensor.flat<int64>();
+      for (int i = 0; i < input_tensor.NumElements(); ++i) {
+        output->at(i) = tensor_values(i);
+      }
+    }
+    TokenizeVector(output.get());
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+  }
+
+  return absl::OkStatus();
+}
+
+void TensorToVectorIntCalculator::TokenizeVector(
+    std::vector<int64>* vector) const {
+  if (!options_.tensor_is_token()) {
+    return;
+  }
+  std::vector<int64> tokens;
+  for (int i = 0; i < vector->size(); ++i) {
+    if (vector->at(i) > options_.token_threshold()) {
+      tokens.push_back(i + 1);
+    }
+  }
+  vector->swap(tokens);
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.proto

@@ -0,0 +1,39 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message TensorToVectorIntCalculatorOptions {
+  extend mediapipe.CalculatorOptions {
+    optional TensorToVectorIntCalculatorOptions ext = 464933130;
+  }
+
+  // If true, unpack a 2d tensor (matrix) into a vector<vector<float>>. If
+  // false, convert a 1d tensor (vector) into a vector<float>.
+  optional bool tensor_is_2d = 1 [default = false];
+
+  // If true, an N-D tensor will be flattened to a vector<float>. This is
+  // exclusive with tensor_is_2d.
+  optional bool flatten_nd = 2 [default = false];
+
+  // If true, represents the vector as tokens and outputs just the position
+  // of values above the threshold into the output vector.
+  optional bool tensor_is_token = 3 [default = false];
+  // Threshold for the token generation.
+  optional float token_threshold = 4 [default = 0.5];
+}

mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_test.cc

@@ -0,0 +1,192 @@
+// Copyright 2018 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+
+namespace mediapipe {
+
+namespace {
+
+namespace tf = ::tensorflow;
+
+class TensorToVectorIntCalculatorTest : public ::testing::Test {
+ protected:
+  void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd,
+                   const bool tensor_is_token = false) {
+    CalculatorGraphConfig::Node config;
+    config.set_calculator("TensorToVectorIntCalculator");
+    config.add_input_stream("input_tensor");
+    config.add_output_stream("output_tensor");
+    auto options = config.mutable_options()->MutableExtension(
+        TensorToVectorIntCalculatorOptions::ext);
+    options->set_tensor_is_2d(tensor_is_2d);
+    options->set_flatten_nd(flatten_nd);
+    options->set_tensor_is_token(tensor_is_token);
+    runner_ = absl::make_unique<CalculatorRunner>(config);
+  }
+
+  std::unique_ptr<CalculatorRunner> runner_;
+};
+
+TEST_F(TensorToVectorIntCalculatorTest, ConvertsToVectorInt) {
+  SetUpRunner(false, false);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT64, tensor_shape);
+  auto tensor_vec = tensor->vec<int64>();
+  for (int i = 0; i < 5; ++i) {
+    // 2^i can be represented exactly in floating point numbers if 'i' is small.
+    tensor_vec(i) = static_cast<int64>(1 << i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  ASSERT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<int64>& output_vector =
+      output_packets[0].Get<std::vector<int64>>();
+
+  EXPECT_EQ(5, output_vector.size());
+  for (int i = 0; i < 5; ++i) {
+    const int64 expected = static_cast<int64>(1 << i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+TEST_F(TensorToVectorIntCalculatorTest, ConvertsToVectorFromInt32) {
+  SetUpRunner(false, false);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT32, tensor_shape);
+  auto tensor_vec = tensor->vec<int32>();
+  for (int i = 0; i < 5; ++i) {
+    // 2^i can be represented exactly in floating point numbers if 'i' is small.
+    tensor_vec(i) = static_cast<int32>(1 << i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  ASSERT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<int64>& output_vector =
+      output_packets[0].Get<std::vector<int64>>();
+
+  EXPECT_EQ(5, output_vector.size());
+  for (int i = 0; i < 5; ++i) {
+    const int64 expected = static_cast<int64>(1 << i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+TEST_F(TensorToVectorIntCalculatorTest, ConvertsToVectorToken) {
+  SetUpRunner(false, false, true);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT32, tensor_shape);
+  auto tensor_vec = tensor->vec<int32>();
+  tensor_vec(0) = 0;
+  tensor_vec(1) = 0;
+  tensor_vec(2) = 1;
+  tensor_vec(3) = 1;
+  tensor_vec(4) = 0;
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  ASSERT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<int64>& output_vector =
+      output_packets[0].Get<std::vector<int64>>();
+
+  EXPECT_EQ(2, output_vector.size());
+  EXPECT_EQ(3, output_vector[0]);
+  EXPECT_EQ(4, output_vector[1]);
+}
+
+TEST_F(TensorToVectorIntCalculatorTest, ConvertsBatchedToVectorVectorInt) {
+  SetUpRunner(true, false);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{1, 5});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT64, tensor_shape);
+  auto slice = tensor->Slice(0, 1).flat<int64>();
+  for (int i = 0; i < 5; ++i) {
+    // 2^i can be represented exactly in floating point numbers if 'i' is small.
+    slice(i) = static_cast<int64>(1 << i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<std::vector<int64>>& output_vectors =
+      output_packets[0].Get<std::vector<std::vector<int64>>>();
+  ASSERT_EQ(1, output_vectors.size());
+  const std::vector<int64>& output_vector = output_vectors[0];
+  EXPECT_EQ(5, output_vector.size());
+  for (int i = 0; i < 5; ++i) {
+    const int64 expected = static_cast<int64>(1 << i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+TEST_F(TensorToVectorIntCalculatorTest, FlattenShouldTakeAllDimensions) {
+  SetUpRunner(false, true);
+  const tf::TensorShape tensor_shape(std::vector<tf::int64>{2, 2, 2});
+  auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT64, tensor_shape);
+  auto slice = tensor->flat<int64>();
+  for (int i = 0; i < 2 * 2 * 2; ++i) {
+    // 2^i can be represented exactly in floating point numbers if 'i' is small.
+    slice(i) = static_cast<int64>(1 << i);
+  }
+
+  const int64 time = 1234;
+  runner_->MutableInputs()->Index(0).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  EXPECT_TRUE(runner_->Run().ok());
+  const std::vector<Packet>& output_packets =
+      runner_->Outputs().Index(0).packets;
+  EXPECT_EQ(1, output_packets.size());
+  EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+  const std::vector<int64>& output_vector =
+      output_packets[0].Get<std::vector<int64>>();
+  EXPECT_EQ(2 * 2 * 2, output_vector.size());
+  for (int i = 0; i < 2 * 2 * 2; ++i) {
+    const int64 expected = static_cast<int64>(1 << i);
+    EXPECT_EQ(expected, output_vector[i]);
+  }
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -505,11 +505,13 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
               << keyed_tensors.first;
         }
       } else {
-        // Pad by replicating the first tensor, then ignore the values.
-        keyed_tensors.second.resize(options_.batch_size());
-        std::fill(keyed_tensors.second.begin() +
-                      inference_state->batch_timestamps_.size(),
-                  keyed_tensors.second.end(), keyed_tensors.second[0]);
+        if (options_.pad_to_batch_size()) {
+          // Pad by replicating the first tensor, then ignore the values.
+          keyed_tensors.second.resize(options_.batch_size());
+          std::fill(keyed_tensors.second.begin() +
+                        inference_state->batch_timestamps_.size(),
+                    keyed_tensors.second.end(), keyed_tensors.second[0]);
+        }
         tf::Tensor concated;
         const tf::Status concat_status =
             tf::tensor::Concat(keyed_tensors.second, &concated);
@@ -576,7 +578,11 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
 
     absl::WriterMutexLock l(&mutex_);
     // Set that we want to split on each index of the 0th dimension.
-    std::vector<tf::int64> split_vector(options_.batch_size(), 1);
+    std::vector<tf::int64> split_vector(
+        options_.pad_to_batch_size()
+            ? options_.batch_size()
+            : inference_state->batch_timestamps_.size(),
+        1);
     for (int i = 0; i < output_tensor_names.size(); ++i) {
       if (options_.batch_size() == 1) {
         if (cc->Outputs().HasTag(output_name_in_signature[i])) {

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto

@@ -49,6 +49,13 @@ message TensorFlowInferenceCalculatorOptions {
   // dimension needs to be added.
   optional bool add_batch_dim_to_tensors = 3 [default = true];
 
+  // Whether to pad the last batch to batch_size or run inference on a partial
+  // batch.
+  // Setting this to false is useful for TPU models that use in-graph batching
+  // as padding in MediaPipe conflicts with merging of batches in tensorflows
+  // batch ops.
+  optional bool pad_to_batch_size = 8 [default = true];
+
   // These pairs represent feed and fetch tensors for handling recurrent state.
   // Each entry is a colon separated pair of strings. The first half of each
   // string is the signature tag for the feed tensor for recurrent state. The

mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc

@@ -458,6 +458,46 @@ TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputed) {
                    ->Get());
 }
 
+TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputedNoPadding) {
+  CalculatorGraphConfig::Node config;
+  config.set_calculator("TensorFlowInferenceCalculator");
+  config.add_input_stream("A:tensor_a");
+  config.add_input_stream("B:tensor_b");
+  config.add_output_stream("MULTIPLIED:tensor_o1");
+  config.add_input_side_packet("SESSION:session");
+  CalculatorOptions options;
+  options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext)
+      ->set_batch_size(3);
+  options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext)
+      ->set_pad_to_batch_size(false);
+  options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext)
+      ->set_add_batch_dim_to_tensors(true);
+  *config.mutable_options() = options;
+
+  runner_ = absl::make_unique<CalculatorRunner>(config);
+  AddSessionInputSidePacket();
+  AddVectorToInputsAsTensor({2, 2, 2}, "A", 0);
+  AddVectorToInputsAsTensor({3, 4, 5}, "B", 0);
+  AddVectorToInputsAsTensor({3, 3, 3}, "A", 1);
+  AddVectorToInputsAsTensor({3, 4, 5}, "B", 1);
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& output_packets_mult =
+      runner_->Outputs().Tag(kMultipliedTag).packets;
+  ASSERT_EQ(2, output_packets_mult.size());
+  const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
+  auto expected_tensor = tf::test::AsTensor<int32>({6, 8, 10});
+  tf::test::ExpectTensorEqual<int32>(tensor_mult, expected_tensor);
+  const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get<tf::Tensor>();
+  auto expected_tensor1 = tf::test::AsTensor<int32>({9, 12, 15});
+  tf::test::ExpectTensorEqual<int32>(tensor_mult1, expected_tensor1);
+
+  EXPECT_EQ(2, runner_
+                   ->GetCounter(
+                       "TensorFlowInferenceCalculator-TotalProcessedTimestamps")
+                   ->Get());
+}
+
 TEST_F(TensorflowInferenceCalculatorTest, GetBatchComputed_MaxInFlight) {
   CalculatorGraphConfig::Node config;
   config.set_calculator("TensorFlowInferenceCalculator");

mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc

@@ -69,7 +69,7 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
         TensorFlowSessionFromFrozenGraphGeneratorOptions::ext);
     bool has_exactly_one_model =
         !options.graph_proto_path().empty()
-            ? !(input_side_packets->HasTag(kStringModelTag) |
+            ? !(input_side_packets->HasTag(kStringModelTag) ||
                 input_side_packets->HasTag(kStringModelFilePathTag))
             : (input_side_packets->HasTag(kStringModelTag) ^
                input_side_packets->HasTag(kStringModelFilePathTag));

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc

@@ -190,6 +190,7 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
     // Copy the packet to copy the otherwise inaccessible shared ptr.
     example_packet_holder_ = cc->InputSidePackets().Tag(kSequenceExampleTag);
     sequence_ = &example_packet_holder_.Get<tf::SequenceExample>();
+    const auto& options = cc->Options<UnpackMediaSequenceCalculatorOptions>();
 
     // Collect the timestamps for all streams keyed by the timestamp feature's
     // key. While creating this data structure we also identify the last
@@ -210,6 +211,13 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
               << "Timestamps must be sequential. If you're seeing this message "
               << "you may have added images to the same SequenceExample twice. "
               << "Key: " << map_kv.first;
+          if (options.output_poststream_as_prestream() &&
+              next_timestamp == Timestamp::PostStream().Value()) {
+            RET_CHECK_EQ(i, 0)
+                << "Detected PostStream() and timestamps being output for the "
+                << "same stream. This is currently invalid.";
+            next_timestamp = Timestamp::PreStream().Value();
+          }
           timestamps_[map_kv.first].push_back(next_timestamp);
           recent_timestamp = next_timestamp;
           if (recent_timestamp < first_timestamp_seen_) {
@@ -247,7 +255,6 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
     process_poststream_ = false;
 
     // Determine the data path and output it.
-    const auto& options = cc->Options<UnpackMediaSequenceCalculatorOptions>();
     const auto& sequence = cc->InputSidePackets()
                                .Tag(kSequenceExampleTag)
                                .Get<tensorflow::SequenceExample>();
@@ -379,10 +386,14 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
       for (int i = 0; i < map_kv.second.size(); ++i) {
         if (map_kv.second[i] >= start_timestamp &&
             map_kv.second[i] < end_timestamp) {
-          const Timestamp current_timestamp =
-              map_kv.second[i] == Timestamp::PostStream().Value()
-                  ? Timestamp::PostStream()
-                  : Timestamp(map_kv.second[i]);
+          Timestamp current_timestamp;
+          if (map_kv.second[i] == Timestamp::PostStream().Value()) {
+            current_timestamp = Timestamp::PostStream();
+          } else if (map_kv.second[i] == Timestamp::PreStream().Value()) {
+            current_timestamp = Timestamp::PreStream();
+          } else {
+            current_timestamp = Timestamp(map_kv.second[i]);
+          }
 
           if (absl::StrContains(map_kv.first, mpms::GetImageTimestampKey())) {
             std::vector<std::string> pieces = absl::StrSplit(map_kv.first, '/');

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto

@@ -56,4 +56,8 @@ message UnpackMediaSequenceCalculatorOptions {
   // the clip start and end times and outputs these for the
   // AudioDecoderCalculator to consume.
   optional AudioDecoderOptions base_audio_decoder_options = 9;
+
+  // Often if a post-stream packet is stored in a SequenceExample, it should be
+  // used as a pre-stream packet in a subsequent graph.
+  optional bool output_poststream_as_prestream = 12;
 }

mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc

@@ -505,6 +505,42 @@ TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) {
               ::testing::Eq(Timestamp::PostStream()));
 }
 
+TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListAtPre) {
+  CalculatorOptions options;
+  options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext)
+      ->set_output_poststream_as_prestream(true);
+  SetUpCalculator({"FLOAT_FEATURE_FDENSE_MAX:max"}, {}, {}, &options);
+  auto input_sequence = absl::make_unique<tf::SequenceExample>();
+  std::string test_video_id = "test_video_id";
+  mpms::SetClipMediaId(test_video_id, input_sequence.get());
+
+  std::string test_image_string = "test_image_string";
+
+  int num_images = 1;
+  for (int i = 0; i < num_images; ++i) {
+    mpms::AddImageTimestamp(i, input_sequence.get());
+    mpms::AddImageEncoded(test_image_string, input_sequence.get());
+  }
+
+  mpms::AddFeatureFloats("FDENSE_MAX", {3.0f, 4.0f}, input_sequence.get());
+  mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(),
+                            input_sequence.get());
+
+  runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+      Adopt(input_sequence.release());
+
+  MP_ASSERT_OK(runner_->Run());
+
+  const std::vector<Packet>& fdense_max_packets =
+      runner_->Outputs().Tag(kFloatFeatureFdenseMaxTag).packets;
+  ASSERT_EQ(fdense_max_packets.size(), 1);
+  const auto& fdense_max_vector =
+      fdense_max_packets[0].Get<std::vector<float>>();
+  ASSERT_THAT(fdense_max_vector, ::testing::ElementsAreArray({3.0f, 4.0f}));
+  ASSERT_THAT(fdense_max_packets[0].Timestamp(),
+              ::testing::Eq(Timestamp::PreStream()));
+}
+
 TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromPacket) {
   SetUpCalculator({}, {"DATA_PATH:data_path"}, {"DATASET_ROOT:root"});
 

mediapipe/calculators/util/BUILD

@@ -81,6 +81,7 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "latency_proto",
     srcs = ["latency.proto"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_proto_library(
@@ -96,11 +97,13 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_frequency_proto",
     srcs = ["packet_frequency.proto"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_proto_library(
     name = "packet_frequency_calculator_proto",
     srcs = ["packet_frequency_calculator.proto"],
+    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",
@@ -110,6 +113,7 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_latency_calculator_proto",
     srcs = ["packet_latency_calculator.proto"],
+    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
         "//mediapipe/framework:calculator_proto",

mediapipe/calculators/util/detection_transformation_calculator.cc

@@ -236,15 +236,23 @@ class DetectionTransformationCalculator : public Node {
           [&](const std::vector<Detection>& detection_vector) {
             return detection_vector;
           });
+      if (transformed_detections.empty()) {
+        OutputEmptyDetections(cc);
+        return absl::OkStatus();
+      }
       ASSIGN_OR_RETURN(input_location_data_format,
                        GetLocationDataFormat(transformed_detections));
       for (Detection& detection : transformed_detections) {
         MP_RETURN_IF_ERROR(ConvertBoundingBox(image_size, &detection));
       }
     } else {
+      Detection transformed_detection(kInDetection(cc).Get());
+      if (!transformed_detection.has_location_data()) {
+        OutputEmptyDetections(cc);
+        return absl::OkStatus();
+      }
       ASSIGN_OR_RETURN(input_location_data_format,
                        GetLocationDataFormat(kInDetection(cc).Get()));
-      Detection transformed_detection(kInDetection(cc).Get());
       MP_RETURN_IF_ERROR(
           ConvertBoundingBox(image_size, &transformed_detection));
       transformed_detections.push_back(transformed_detection);
@@ -288,6 +296,27 @@ class DetectionTransformationCalculator : public Node {
   }
 
  private:
+  void OutputEmptyDetections(CalculatorContext* cc) {
+    if (kOutPixelDetection(cc).IsConnected()) {
+      kOutPixelDetection(cc).Send(Detection());
+    }
+    if (kOutPixelDetections(cc).IsConnected()) {
+      kOutPixelDetections(cc).Send(std::vector<Detection>());
+    }
+    if (kOutPixelDetectionList(cc).IsConnected()) {
+      kOutPixelDetectionList(cc).Send(DetectionList());
+    }
+    if (kOutRelativeDetection(cc).IsConnected()) {
+      kOutRelativeDetection(cc).Send(Detection());
+    }
+    if (kOutRelativeDetections(cc).IsConnected()) {
+      kOutRelativeDetections(cc).Send(std::vector<Detection>());
+    }
+    if (kOutRelativeDetectionList(cc).IsConnected()) {
+      kOutRelativeDetectionList(cc).Send(DetectionList());
+    }
+  }
+
   bool output_relative_bounding_boxes_;
   bool output_pixel_bounding_boxes_;
 };

mediapipe/calculators/util/latency.proto

@@ -1,6 +1,7 @@
 // Proto messages related to latency measurement for Soapbox.
 syntax = "proto2";
 
+// TODO: Switch to package mediapipe.
 package mediapipe;
 
 // Contains the latency information for a packet stream in mediapipe. The

mediapipe/calculators/util/packet_frequency.proto

@@ -1,5 +1,6 @@
 syntax = "proto2";
 
+// TODO: Switch to package mediapipe.
 package mediapipe;
 
 // Contains the packet frequency information.

mediapipe/calculators/util/packet_frequency_calculator.proto

@@ -14,6 +14,7 @@
 
 syntax = "proto2";
 
+// TODO: Switch to package mediapipe.
 package mediapipe;
 
 import "mediapipe/framework/calculator.proto";

mediapipe/calculators/util/packet_latency_calculator.proto

@@ -14,6 +14,7 @@
 
 syntax = "proto2";
 
+// TODO: Switch to package mediapipe.
 package mediapipe;
 
 import "mediapipe/framework/calculator.proto";

mediapipe/examples/desktop/autoflip/BUILD

@@ -16,13 +16,22 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
 
 licenses(["notice"])
 
-package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+package(default_visibility = [
+    "//mediapipe/examples:__subpackages__",
+])
 
 proto_library(
     name = "autoflip_messages_proto",
     srcs = ["autoflip_messages.proto"],
     deps = [
-        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+java_lite_proto_library(
+    name = "autoflip_messages_java_proto_lite",
+    deps = [
+        ":autoflip_messages_proto",
     ],
 )
 
@@ -38,6 +47,9 @@ mediapipe_cc_proto_library(
 
 cc_binary(
     name = "run_autoflip",
+    data = [
+        "//mediapipe/modules/face_detection:face_detection_full_range_sparse.tflite",
+    ],
     deps = [
         "//mediapipe/calculators/core:packet_thinner_calculator",
         "//mediapipe/calculators/image:scale_image_calculator",

mediapipe/examples/desktop/autoflip/autoflip_messages.proto

@@ -17,7 +17,9 @@ syntax = "proto2";
 
 package mediapipe.autoflip;
 
-import "mediapipe/framework/calculator.proto";
+import "mediapipe/framework/calculator_options.proto";
+
+option java_multiple_files = true;
 
 // Borders detected on the frame as well as non-border color (if present).
 // Next tag: 4

mediapipe/examples/desktop/autoflip/calculators/BUILD

@@ -289,7 +289,6 @@ cc_library(
         ":signal_fusing_calculator_cc_proto",
         "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@com_google_absl//absl/container:btree",
@@ -343,7 +342,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":shot_boundary_calculator_cc_proto",
-        "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:image_frame",
@@ -358,10 +356,7 @@ cc_library(
 proto_library(
     name = "shot_boundary_calculator_proto",
     srcs = ["shot_boundary_calculator.proto"],
-    deps = [
-        "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto",
-        "//mediapipe/framework:calculator_proto",
-    ],
+    deps = ["//mediapipe/framework:calculator_proto"],
 )
 
 mediapipe_cc_proto_library(
@@ -414,7 +409,6 @@ cc_library(
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "@com_google_absl//absl/memory",
     ],
     alwayslink = 1,
 )
@@ -452,7 +446,6 @@ cc_test(
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/formats:location_data_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:ret_check",
@@ -505,7 +498,6 @@ cc_test(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:detection_cc_proto",
-        "//mediapipe/framework/formats:location_data_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:ret_check",

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc

@@ -203,6 +203,7 @@ absl::Status ContentZoomingCalculator::GetContract(
 }
 
 absl::Status ContentZoomingCalculator::Open(mediapipe::CalculatorContext* cc) {
+  cc->SetOffset(mediapipe::TimestampDiff(0));
   options_ = cc->Options<ContentZoomingCalculatorOptions>();
   if (options_.has_kinematic_options()) {
     return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)

mediapipe/examples/desktop/autoflip/quality/BUILD

@@ -16,7 +16,9 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
 
 licenses(["notice"])
 
-package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+package(default_visibility = [
+    "//mediapipe/examples:__subpackages__",
+])
 
 proto_library(
     name = "cropping_proto",

mediapipe/examples/desktop/face_detection/BUILD

@@ -18,6 +18,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "face_detection_full_range_cpu",
+    data = ["//mediapipe/modules/face_detection:face_detection_full_range.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_deps",
@@ -26,6 +27,7 @@ cc_binary(
 
 cc_binary(
     name = "face_detection_cpu",
+    data = ["//mediapipe/modules/face_detection:face_detection_short_range.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/face_detection:desktop_live_calculators",
@@ -35,6 +37,7 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "face_detection_gpu",
+    data = ["//mediapipe/modules/face_detection:face_detection_short_range.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/face_detection:desktop_live_gpu_calculators",

mediapipe/examples/desktop/face_mesh/BUILD

@@ -18,6 +18,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "face_mesh_tflite",
+    data = ["//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite"],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/face_mesh:desktop_calculators",
@@ -26,6 +27,7 @@ cc_binary(
 
 cc_binary(
     name = "face_mesh_cpu",
+    data = ["//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/face_mesh:desktop_live_calculators",
@@ -35,6 +37,7 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "face_mesh_gpu",
+    data = ["//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators",

mediapipe/examples/desktop/hair_segmentation/BUILD

@@ -19,6 +19,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 # Linux only
 cc_binary(
     name = "hair_segmentation_gpu",
+    data = ["//mediapipe/models:hair_segmentation.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/hair_segmentation:mobile_calculators",
@@ -27,6 +28,7 @@ cc_binary(
 
 cc_binary(
     name = "hair_segmentation_cpu",
+    data = ["//mediapipe/models:hair_segmentation.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
     ] + select({

mediapipe/examples/desktop/hand_tracking/BUILD

@@ -18,6 +18,10 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "hand_tracking_tflite",
+    data = [
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/hand_tracking:desktop_tflite_calculators",
@@ -26,6 +30,10 @@ cc_binary(
 
 cc_binary(
     name = "hand_tracking_cpu",
+    data = [
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/hand_tracking:desktop_tflite_calculators",
@@ -35,6 +43,10 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "hand_tracking_gpu",
+    data = [
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/palm_detection:palm_detection_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/hand_tracking:mobile_calculators",

mediapipe/examples/desktop/holistic_tracking/BUILD

@@ -18,6 +18,13 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "holistic_tracking_cpu",
+    data = [
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/holistic_landmark:hand_recrop.tflite",
+        "//mediapipe/modules/pose_detection:pose_detection.tflite",
+        "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/holistic_tracking:holistic_tracking_cpu_graph_deps",
@@ -27,6 +34,13 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "holistic_tracking_gpu",
+    data = [
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite",
+        "//mediapipe/modules/holistic_landmark:hand_recrop.tflite",
+        "//mediapipe/modules/pose_detection:pose_detection.tflite",
+        "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu_deps",

mediapipe/examples/desktop/iris_tracking/BUILD

@@ -19,6 +19,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 cc_binary(
     name = "iris_depth_from_image_desktop",
     srcs = ["iris_depth_from_image_desktop.cc"],
+    data = ["//mediapipe/modules/iris_landmark:iris_landmark.tflite"],
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
@@ -37,6 +38,7 @@ cc_binary(
 
 cc_binary(
     name = "iris_tracking_cpu_video_input",
+    data = ["//mediapipe/modules/iris_landmark:iris_landmark.tflite"],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_video_input_deps",
@@ -45,6 +47,7 @@ cc_binary(
 
 cc_binary(
     name = "iris_tracking_cpu",
+    data = ["//mediapipe/modules/iris_landmark:iris_landmark.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps",
@@ -54,6 +57,7 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "iris_tracking_gpu",
+    data = ["//mediapipe/modules/iris_landmark:iris_landmark.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps",

mediapipe/examples/desktop/object_detection/BUILD

@@ -18,6 +18,10 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "object_detection_tensorflow",
+    data = [
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/object_detection:desktop_tensorflow_calculators",
@@ -28,6 +32,10 @@ cc_binary(
 
 cc_binary(
     name = "object_detection_tflite",
+    data = [
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
@@ -36,6 +44,10 @@ cc_binary(
 
 cc_binary(
     name = "object_detection_cpu",
+    data = [
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/object_detection:desktop_tflite_calculators",

mediapipe/examples/desktop/object_detection_3d/BUILD

@@ -27,6 +27,15 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 # Cup: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_cup.tflite,allowed_labels=Mug
 cc_binary(
     name = "objectron_cpu",
+    data = [
+        "//mediapipe/modules/objectron:object_detection_3d_camera.tflite",
+        "//mediapipe/modules/objectron:object_detection_3d_chair.tflite",
+        "//mediapipe/modules/objectron:object_detection_3d_chair_1stage.tflite",
+        "//mediapipe/modules/objectron:object_detection_3d_cup.tflite",
+        "//mediapipe/modules/objectron:object_detection_3d_sneakers.tflite",
+        "//mediapipe/modules/objectron:object_detection_3d_sneakers_1stage.tflite",
+        "//mediapipe/modules/objectron:object_detection_ssd_mobilenetv2_oidv4_fp16.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/object_detection_3d:desktop_cpu_calculators",

mediapipe/examples/desktop/object_tracking/BUILD

@@ -18,6 +18,10 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "object_tracking_cpu",
+    data = [
+        "//mediapipe/models:ssdlite_object_detection.tflite",
+        "//mediapipe/models:ssdlite_object_detection_labelmap.txt",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/tracking:desktop_calculators",

mediapipe/examples/desktop/pose_tracking/BUILD

@@ -18,6 +18,10 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "pose_tracking_cpu",
+    data = [
+        "//mediapipe/modules/pose_detection:pose_detection.tflite",
+        "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/pose_tracking:pose_tracking_cpu_deps",
@@ -27,6 +31,10 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "pose_tracking_gpu",
+    data = [
+        "//mediapipe/modules/pose_detection:pose_detection.tflite",
+        "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite",
+    ],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/pose_tracking:pose_tracking_gpu_deps",

mediapipe/examples/desktop/selfie_segmentation/BUILD

@@ -18,6 +18,7 @@ package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
     name = "selfie_segmentation_cpu",
+    data = ["//mediapipe/modules/selfie_segmentation:selfie_segmentation.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_cpu_deps",
@@ -27,6 +28,7 @@ cc_binary(
 # Linux only
 cc_binary(
     name = "selfie_segmentation_gpu",
+    data = ["//mediapipe/modules/selfie_segmentation:selfie_segmentation.tflite"],
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
         "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu_deps",

mediapipe/examples/ios/common/BUILD

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-licenses(["notice"])  # Apache 2.0
+licenses(["notice"])
 
 objc_library(
     name = "CommonMediaPipeAppLibrary",

mediapipe/examples/ios/objectdetectiongpu/BUILD

@@ -22,7 +22,7 @@ load(
     "example_provisioning",
 )
 
-licenses(["notice"])  # Apache 2.0
+licenses(["notice"])
 
 MIN_IOS_VERSION = "11.0"
 

mediapipe/framework/BUILD

@@ -102,13 +102,13 @@ mediapipe_proto_library(
 mediapipe_proto_library(
     name = "packet_factory_proto",
     srcs = ["packet_factory.proto"],
-    visibility = [":mediapipe_internal"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_proto_library(
     name = "packet_generator_proto",
     srcs = ["packet_generator.proto"],
-    visibility = [":mediapipe_internal"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_proto_library(
@@ -153,6 +153,7 @@ mediapipe_proto_library(
     deps = ["//mediapipe/framework:mediapipe_options_proto"],
 )
 
+# It is for pure-native Android builds where the library can't have any dependency on libandroid.so
 config_setting(
     name = "android_no_jni",
     define_values = {"MEDIAPIPE_NO_JNI": "1"},

mediapipe/framework/api2/port.h

@@ -460,6 +460,9 @@ class OutputShardAccessBase {
   OutputShardAccessBase(const CalculatorContext& cc, OutputStreamShard* output)
       : context_(cc), output_(output) {}
 
+  Timestamp NextTimestampBound() const {
+    return (output_) ? output_->NextTimestampBound() : Timestamp::Unset();
+  }
   void SetNextTimestampBound(Timestamp timestamp) {
     if (output_) output_->SetNextTimestampBound(timestamp);
   }

mediapipe/framework/calculator_profile.proto

@@ -20,7 +20,6 @@ import "mediapipe/framework/calculator.proto";
 
 option java_package = "com.google.mediapipe.proto";
 option java_outer_classname = "CalculatorProfileProto";
-option objc_class_prefix = "MediaPipe";
 
 // Stores the profiling information.
 //

mediapipe/framework/deps/BUILD

@@ -41,7 +41,10 @@ bzl_library(
 proto_library(
     name = "proto_descriptor_proto",
     srcs = ["proto_descriptor.proto"],
-    visibility = ["//mediapipe/framework:__subpackages__"],
+    visibility = [
+        "//mediapipe/deps:__subpackages__",
+        "//mediapipe/framework:__subpackages__",
+    ],
 )
 
 mediapipe_cc_proto_library(

mediapipe/framework/deps/proto_descriptor.proto

@@ -2,6 +2,9 @@ syntax = "proto2";
 
 package mediapipe;
 
+option java_package = "com.google.mediapipe.proto";
+option java_outer_classname = "FieldDescriptorProtoProto";
+
 // Describes a field within a message.
 message FieldDescriptorProto {
   enum Type {

mediapipe/framework/deps/registration.h

@@ -129,8 +129,8 @@ namespace mediapipe {
 //      }));
 
 namespace registration_internal {
-constexpr char kCxxSep[] = "::";
-constexpr char kNameSep[] = ".";
+inline constexpr char kCxxSep[] = "::";
+inline constexpr char kNameSep[] = ".";
 
 template <typename T>
 struct WrapStatusOr {
@@ -245,7 +245,7 @@ class FunctionRegistry {
   // The name must be either unqualified or fully qualified with a leading "::".
   // The leading "::" in a fully qualified name is stripped.
   std::string GetNormalizedName(const std::string& name) {
-    constexpr auto kCxxSep = registration_internal::kCxxSep;
+    using ::mediapipe::registration_internal::kCxxSep;
     std::vector<std::string> names = absl::StrSplit(name, kCxxSep);
     if (names[0].empty()) {
       names.erase(names.begin());
@@ -261,8 +261,8 @@ class FunctionRegistry {
   // Namespaces are separated by kNameSep.
   std::string GetQualifiedName(const std::string& ns,
                                const std::string& name) const {
-    constexpr auto kCxxSep = registration_internal::kCxxSep;
-    constexpr auto kNameSep = registration_internal::kNameSep;
+    using ::mediapipe::registration_internal::kCxxSep;
+    using ::mediapipe::registration_internal::kNameSep;
     std::vector<std::string> names = absl::StrSplit(name, kNameSep);
     if (names[0].empty()) {
       names.erase(names.begin());
@@ -291,7 +291,7 @@ class FunctionRegistry {
 
   // For names included in NamespaceAllowlist, strips the namespace.
   std::string GetAdjustedName(const std::string& name) {
-    constexpr auto kCxxSep = registration_internal::kCxxSep;
+    using ::mediapipe::registration_internal::kCxxSep;
     std::vector<std::string> names = absl::StrSplit(name, kCxxSep);
     std::string base_name = names.back();
     names.pop_back();

mediapipe/framework/deps/status_matchers.h

@@ -135,6 +135,123 @@ IsOkAndHoldsMatcher<typename std::decay<InnerMatcher>::type> IsOkAndHolds(
 // Returns a gMock matcher that matches a Status or StatusOr<> which is OK.
 inline IsOkMatcher IsOk() { return IsOkMatcher(); }
 
+////////////////////////////////////////////////////////////
+// Implementation of StatusIs().
+//
+// StatusIs() is a polymorphic matcher. This class is the common
+// implementation of it shared by all types T where StatusIs() can be used as
+// a Matcher<T>.
+
+class StatusIsMatcherCommonImpl {
+ public:
+  StatusIsMatcherCommonImpl(
+      ::testing::Matcher<const absl::StatusCode> code_matcher,
+      ::testing::Matcher<const std::string&> message_matcher)
+      : code_matcher_(std::move(code_matcher)),
+        message_matcher_(std::move(message_matcher)) {}
+
+  void DescribeTo(std::ostream* os) const {
+    *os << "has a status code that ";
+    code_matcher_.DescribeTo(os);
+    *os << ", and has an error message that ";
+    message_matcher_.DescribeTo(os);
+  }
+
+  void DescribeNegationTo(std::ostream* os) const {
+    *os << "has a status code that ";
+    code_matcher_.DescribeNegationTo(os);
+    *os << ", or has an error message that ";
+    message_matcher_.DescribeNegationTo(os);
+  }
+
+  bool MatchAndExplain(const absl::Status& status,
+                       ::testing::MatchResultListener* result_listener) const {
+    ::testing::StringMatchResultListener inner_listener;
+
+    inner_listener.Clear();
+    if (!code_matcher_.MatchAndExplain(status.code(), &inner_listener)) {
+      *result_listener << (inner_listener.str().empty()
+                               ? "whose status code is wrong"
+                               : "which has a status code " +
+                                     inner_listener.str());
+      return false;
+    }
+
+    if (!message_matcher_.Matches(std::string(status.message()))) {
+      *result_listener << "whose error message is wrong";
+      return false;
+    }
+
+    return true;
+  }
+
+ private:
+  const ::testing::Matcher<const absl::StatusCode> code_matcher_;
+  const ::testing::Matcher<const std::string&> message_matcher_;
+};
+
+// Monomorphic implementation of matcher StatusIs() for a given type T. T can
+// be Status, StatusOr<>, or a reference to either of them.
+template <typename T>
+class MonoStatusIsMatcherImpl : public ::testing::MatcherInterface<T> {
+ public:
+  explicit MonoStatusIsMatcherImpl(StatusIsMatcherCommonImpl common_impl)
+      : common_impl_(std::move(common_impl)) {}
+
+  void DescribeTo(std::ostream* os) const override {
+    common_impl_.DescribeTo(os);
+  }
+
+  void DescribeNegationTo(std::ostream* os) const override {
+    common_impl_.DescribeNegationTo(os);
+  }
+
+  bool MatchAndExplain(
+      T actual_value,
+      ::testing::MatchResultListener* result_listener) const override {
+    return common_impl_.MatchAndExplain(GetStatus(actual_value),
+                                        result_listener);
+  }
+
+ private:
+  StatusIsMatcherCommonImpl common_impl_;
+};
+
+// Implements StatusIs() as a polymorphic matcher.
+class StatusIsMatcher {
+ public:
+  StatusIsMatcher(::testing::Matcher<const absl::StatusCode> code_matcher,
+                  ::testing::Matcher<const std::string&> message_matcher)
+      : common_impl_(
+            ::testing::MatcherCast<const absl::StatusCode>(code_matcher),
+            ::testing::MatcherCast<const std::string&>(message_matcher)) {}
+
+  // Converts this polymorphic matcher to a monomorphic matcher of the given
+  // type. T can be StatusOr<>, Status, or a reference to either of them.
+  template <typename T>
+  operator ::testing::Matcher<T>() const {  // NOLINT
+    return ::testing::MakeMatcher(new MonoStatusIsMatcherImpl<T>(common_impl_));
+  }
+
+ private:
+  const StatusIsMatcherCommonImpl common_impl_;
+};
+
+// Returns a matcher that matches a Status or StatusOr<> whose status code
+// matches code_matcher, and whose error message matches message_matcher.
+template <typename CodeMatcher, typename MessageMatcher>
+StatusIsMatcher StatusIs(CodeMatcher code_matcher,
+                         MessageMatcher message_matcher) {
+  return StatusIsMatcher(std::move(code_matcher), std::move(message_matcher));
+}
+
+// Returns a matcher that matches a Status or StatusOr<> whose status code
+// matches code_matcher.
+template <typename CodeMatcher>
+StatusIsMatcher StatusIs(CodeMatcher code_matcher) {
+  return StatusIs(std::move(code_matcher), ::testing::_);
+}
+
 }  // namespace mediapipe
 
 // Macros for testing the results of functions that return absl::Status or
@@ -142,4 +259,17 @@ inline IsOkMatcher IsOk() { return IsOkMatcher(); }
 #define MP_EXPECT_OK(expression) EXPECT_THAT(expression, mediapipe::IsOk())
 #define MP_ASSERT_OK(expression) ASSERT_THAT(expression, mediapipe::IsOk())
 
+#define STATUS_MACROS_IMPL_CONCAT_INNER_(x, y) x##y
+#define STATUS_MACROS_IMPL_CONCAT_(x, y) STATUS_MACROS_IMPL_CONCAT_INNER_(x, y)
+
+#undef MP_ASSERT_OK_AND_ASSIGN
+#define MP_ASSERT_OK_AND_ASSIGN(lhs, rexpr) \
+  MP_ASSERT_OK_AND_ASSIGN_IMPL_(            \
+      STATUS_MACROS_IMPL_CONCAT_(_status_or_value, __LINE__), lhs, rexpr)
+
+#define MP_ASSERT_OK_AND_ASSIGN_IMPL_(statusor, lhs, rexpr) \
+  auto statusor = (rexpr);                                  \
+  ASSERT_TRUE(statusor.ok());                               \
+  lhs = std::move(statusor.value())
+
 #endif  // MEDIAPIPE_DEPS_STATUS_MATCHERS_H_

mediapipe/framework/formats/BUILD

@@ -206,12 +206,6 @@ cc_library(
     name = "location",
     srcs = ["location.cc"],
     hdrs = ["location.h"],
-    defines = select({
-        "//conditions:default": [],
-        "//mediapipe:android": ["MEDIAPIPE_ANDROID_OPENCV"],
-        ":portable_opencv": ["MEDIAPIPE_ANDROID_OPENCV"],
-        ":opencv": [],
-    }),
     visibility = ["//visibility:public"],
     deps = [
         "@com_google_protobuf//:protobuf",
@@ -232,11 +226,6 @@ cc_library(
         "//mediapipe/framework/port:statusor",
         "//mediapipe/framework/formats/annotation:rasterization_cc_proto",
     ] + select({
-        "//conditions:default": [
-            "//mediapipe/framework/port:opencv_imgproc",
-        ],
-        "//mediapipe/framework/port:disable_opencv": [],
-    }) + select({
         "//conditions:default": [
         ],
         "//mediapipe:android": [],
@@ -245,6 +234,28 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "location_opencv",
+    srcs = ["location_opencv.cc"],
+    hdrs = ["location_opencv.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":location",
+        "//mediapipe/framework/port:opencv_imgproc",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "location_opencv_test",
+    srcs = ["location_opencv_test.cc"],
+    deps = [
+        ":location_opencv",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:rectangle",
+    ],
+)
+
 cc_library(
     name = "video_stream_header",
     hdrs = ["video_stream_header.h"],
@@ -464,6 +475,7 @@ cc_library(
             "-framework MetalKit",
         ],
         "//conditions:default": [],
+        "//mediapipe/framework:android_no_jni": [],
         "//mediapipe:android": [
             "-landroid",
         ],

mediapipe/framework/formats/affine_transform_data.proto

@@ -16,8 +16,6 @@ syntax = "proto2";
 
 package mediapipe;
 
-option objc_class_prefix = "MediaPipe";
-
 // Proto for serializing Vector2 data
 message Vector2Data {
   optional float x = 1;

mediapipe/framework/formats/annotation/locus.proto

@@ -18,6 +18,8 @@ package mediapipe;
 
 import "mediapipe/framework/formats/annotation/rasterization.proto";
 
+option cc_enable_arenas = true;
+
 // A way to identify a part of an image. A locus does not need to correspond to
 // a subset of pixels -- e.g. for a local descriptor we might define a locus in
 // terms of its location and scale, even if the support of the descriptor is the

mediapipe/framework/formats/classification.proto

@@ -20,7 +20,6 @@ syntax = "proto2";
 
 package mediapipe;
 
-option objc_class_prefix = "MediaPipe";
 option java_package = "com.google.mediapipe.formats.proto";
 option java_outer_classname = "ClassificationProto";
 

mediapipe/framework/formats/image.cc

@@ -42,5 +42,7 @@ bool Image::ConvertToGpu() const {
 
 MEDIAPIPE_REGISTER_TYPE(mediapipe::Image, "::mediapipe::Image", nullptr,
                         nullptr);
+MEDIAPIPE_REGISTER_TYPE(std::vector<mediapipe::Image>,
+                        "::std::vector<::mediapipe::Image>", nullptr, nullptr);
 
 }  // namespace mediapipe

mediapipe/framework/formats/image_format.proto

@@ -23,6 +23,9 @@ syntax = "proto2";
 
 package mediapipe;
 
+option java_package = "com.google.mediapipe.formats.proto";
+option java_outer_classname = "ImageFormatProto";
+
 message ImageFormat {
   enum Format {
     // The format is unknown.  It is not valid for an ImageFrame to be

mediapipe/framework/formats/location.cc

@@ -32,10 +32,6 @@
 #include "mediapipe/framework/tool/status_util.h"
 #include "mediapipe/framework/type_map.h"
 
-#if LOCATION_OPENCV
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#endif
-
 namespace mediapipe {
 
 namespace {
@@ -61,41 +57,6 @@ Rectangle_i MaskToRectangle(const LocationData& location_data) {
   return Rectangle_i(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
 }
 
-#if LOCATION_OPENCV
-std::unique_ptr<cv::Mat> MaskToMat(const LocationData::BinaryMask& mask) {
-  auto image = absl::make_unique<cv::Mat>();
-  *image = cv::Mat::zeros(cv::Size(mask.width(), mask.height()), CV_32FC1);
-  for (const auto& interval : mask.rasterization().interval()) {
-    for (int x = interval.left_x(); x <= interval.right_x(); ++x) {
-      image->at<float>(interval.y(), x) = 1.0f;
-    }
-  }
-  return image;
-}
-absl::StatusOr<std::unique_ptr<cv::Mat>> RectangleToMat(
-    int image_width, int image_height, const Rectangle_i& rect) {
-  // These checks prevent undefined behavior caused when setting memory for
-  // rectangles whose edges lie outside image edges.
-  if (rect.ymin() < 0 || rect.xmin() < 0 || rect.xmax() > image_width ||
-      rect.ymax() > image_height) {
-    return absl::InvalidArgumentError(absl::Substitute(
-        "Rectangle must be bounded by image boundaries.\nImage Width: "
-        "$0\nImage Height: $1\nRectangle: [($2, $3), ($4, $5)]",
-        image_width, image_height, rect.xmin(), rect.ymin(), rect.xmax(),
-        rect.ymax()));
-  }
-  // Allocate image and set pixels of foreground mask.
-  auto image = absl::make_unique<cv::Mat>();
-  *image = cv::Mat::zeros(cv::Size(image_width, image_height), CV_32FC1);
-  for (int y = rect.ymin(); y < rect.ymax(); ++y) {
-    for (int x = rect.xmin(); x < rect.xmax(); ++x) {
-      image->at<float>(y, x) = 1.0f;
-    }
-  }
-  return std::move(image);
-}
-#endif  // OPENCV
-
 }  // namespace
 
 Location::Location() {}
@@ -134,12 +95,6 @@ Location Location::CreateBBoxLocation(const ::mediapipe::BoundingBox& bbox) {
                             bbox.lower_y() - bbox.upper_y());
 }
 
-#if LOCATION_OPENCV
-Location Location::CreateBBoxLocation(const cv::Rect& rect) {
-  return CreateBBoxLocation(rect.x, rect.y, rect.width, rect.height);
-}
-#endif
-
 Location Location::CreateRelativeBBoxLocation(float relative_xmin,
                                               float relative_ymin,
                                               float relative_width,
@@ -159,41 +114,6 @@ Location Location::CreateRelativeBBoxLocation(const Rectangle_f& rect) {
                                     rect.Height());
 }
 
-#if LOCATION_OPENCV
-template <typename T>
-Location Location::CreateCvMaskLocation(const cv::Mat_<T>& mask) {
-  CHECK_EQ(1, mask.channels())
-      << "The specified cv::Mat mask should be single-channel.";
-
-  LocationData location_data;
-  location_data.set_format(LocationData::MASK);
-  location_data.mutable_mask()->set_width(mask.cols);
-  location_data.mutable_mask()->set_height(mask.rows);
-  auto* rasterization = location_data.mutable_mask()->mutable_rasterization();
-  const auto kForegroundThreshold = static_cast<T>(0);
-  for (int y = 0; y < mask.rows; y++) {
-    Rasterization::Interval* interval;
-    bool traversing = false;
-    for (int x = 0; x < mask.cols; x++) {
-      const bool is_foreground =
-          mask.template at<T>(y, x) > kForegroundThreshold;
-      if (is_foreground) {
-        if (!traversing) {
-          interval = rasterization->add_interval();
-          interval->set_y(y);
-          interval->set_left_x(x);
-          traversing = true;
-        }
-        interval->set_right_x(x);
-      } else {
-        traversing = false;
-      }
-    }
-  }
-  return Location(location_data);
-}
-#endif
-
 LocationData::Format Location::GetFormat() const {
   return location_data_.format();
 }
@@ -274,62 +194,6 @@ Location& Location::Scale(const float scale) {
   return *this;
 }
 
-#if LOCATION_OPENCV
-Location& Location::Enlarge(const float factor) {
-  CHECK_GT(factor, 0.0f);
-  if (factor == 1.0f) return *this;
-  switch (location_data_.format()) {
-    case LocationData::GLOBAL: {
-      // Do nothing.
-      break;
-    }
-    case LocationData::BOUNDING_BOX: {
-      auto* box = location_data_.mutable_bounding_box();
-      const int enlarged_int_width =
-          static_cast<int>(std::round(factor * box->width()));
-      const int enlarged_int_height =
-          static_cast<int>(std::round(factor * box->height()));
-      box->set_xmin(
-          std::max(box->xmin() + box->width() / 2 - enlarged_int_width / 2, 0));
-      box->set_ymin(std::max(
-          box->ymin() + box->height() / 2 - enlarged_int_height / 2, 0));
-      box->set_width(enlarged_int_width);
-      box->set_height(enlarged_int_height);
-      break;
-    }
-    case LocationData::RELATIVE_BOUNDING_BOX: {
-      auto* box = location_data_.mutable_relative_bounding_box();
-      box->set_xmin(box->xmin() - ((factor - 1.0) * box->width()) / 2.0);
-      box->set_ymin(box->ymin() - ((factor - 1.0) * box->height()) / 2.0);
-      box->set_width(factor * box->width());
-      box->set_height(factor * box->height());
-      break;
-    }
-    case LocationData::MASK: {
-      auto mask_bounding_box = MaskToRectangle(location_data_);
-      const float scaler = std::fabs(factor - 1.0f);
-      const int dilation_width =
-          static_cast<int>(std::round(scaler * mask_bounding_box.Width()));
-      const int dilation_height =
-          static_cast<int>(std::round(scaler * mask_bounding_box.Height()));
-      if (dilation_width == 0 || dilation_height == 0) break;
-      cv::Mat morph_element(dilation_height, dilation_width, CV_8U,
-                            cv::Scalar(1));
-      auto mask = GetCvMask();
-      if (factor > 1.0f) {
-        cv::dilate(*mask, *mask, morph_element);
-      } else {
-        cv::erode(*mask, *mask, morph_element);
-      }
-      Location::CreateCvMaskLocation<uint8>(*mask).ConvertToProto(
-          &location_data_);
-      break;
-    }
-  }
-  return *this;
-}
-#endif
-
 Location& Location::Square(int image_width, int image_height) {
   switch (location_data_.format()) {
     case LocationData::GLOBAL: {
@@ -615,51 +479,6 @@ template <>
   return bounding_box;
 }
 
-#if LOCATION_OPENCV
-std::unique_ptr<cv::Mat> Location::GetCvMask() const {
-  CHECK_EQ(LocationData::MASK, location_data_.format());
-  const auto& mask = location_data_.mask();
-  std::unique_ptr<cv::Mat> mat(
-      new cv::Mat(mask.height(), mask.width(), CV_8UC1, cv::Scalar(0)));
-  for (const auto& interval :
-       location_data_.mask().rasterization().interval()) {
-    for (int x = interval.left_x(); x <= interval.right_x(); ++x) {
-      mat->at<uint8>(interval.y(), x) = 255;
-    }
-  }
-  return mat;
-}
-
-std::unique_ptr<cv::Mat> Location::ConvertToCvMask(int image_width,
-                                                   int image_height) const {
-  switch (location_data_.format()) {
-    case LocationData::GLOBAL:
-    case LocationData::BOUNDING_BOX:
-    case LocationData::RELATIVE_BOUNDING_BOX: {
-      auto status_or_mat =
-          RectangleToMat(image_width, image_height,
-                         ConvertToBBox<Rectangle_i>(image_width, image_height));
-      if (!status_or_mat.ok()) {
-        LOG(ERROR) << status_or_mat.status().message();
-        return nullptr;
-      }
-      return std::move(status_or_mat).value();
-    }
-    case LocationData::MASK: {
-      return MaskToMat(location_data_.mask());
-    }
-  }
-// This should never happen; a new LocationData::Format enum was introduced
-// without updating this function's switch(...) to support it.
-#if !defined(MEDIAPIPE_MOBILE) && !defined(MEDIAPIPE_LITE)
-  LOG(ERROR) << "Location's LocationData has format not supported by "
-                "Location::ConvertToMask: "
-             << location_data_.DebugString();
-#endif
-  return nullptr;
-}
-#endif
-
 std::vector<Point2_f> Location::GetRelativeKeypoints() const {
   CHECK_EQ(LocationData::RELATIVE_BOUNDING_BOX, location_data_.format());
   std::vector<Point2_f> keypoints;
@@ -703,9 +522,4 @@ LocationData Location::ConvertToProto() const {
   return location_data;
 }
 
-#if LOCATION_OPENCV
-template Location Location::CreateCvMaskLocation(const cv::Mat_<uint8>& mask);
-template Location Location::CreateCvMaskLocation(const cv::Mat_<float>& mask);
-#endif  // LOCATION_OPENCV
-
 }  // namespace mediapipe

mediapipe/framework/formats/location.h

@@ -30,21 +30,6 @@
 #include "mediapipe/framework/port/point2.h"
 #include "mediapipe/framework/port/rectangle.h"
 
-// clang-format off
-#if !defined(LOCATION_OPENCV)
-#  if !MEDIAPIPE_DISABLE_OPENCV && \
-      (!defined(MEDIAPIPE_MOBILE) || defined(MEDIAPIPE_ANDROID_OPENCV))
-#    define LOCATION_OPENCV 1
-#  else
-#    define LOCATION_OPENCV 0
-#  endif
-#endif
-
-#if LOCATION_OPENCV
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#endif
-// clang-format on
-
 namespace mediapipe {
 class BoundingBox;
 }  // namespace mediapipe
@@ -68,9 +53,6 @@ class Location {
   // formats.
   static Location CreateBBoxLocation(const Rectangle_i& rect);
   static Location CreateBBoxLocation(const ::mediapipe::BoundingBox& bbox);
-#if LOCATION_OPENCV
-  static Location CreateBBoxLocation(const cv::Rect& rect);
-#endif
   // Creates a location of type RELATIVE_BOUNDING_BOX, i.e. it is based on a
   // bounding box defined by its upper left corner (xmin, ymin) and its width
   // and height, all relative to the image dimensions.
@@ -81,14 +63,6 @@ class Location {
   // Creates a location of type RELATIVE_BOUNDING_BOX from bounding boxes in
   // various formats.
   static Location CreateRelativeBBoxLocation(const Rectangle_f& relative_rect);
-#if LOCATION_OPENCV
-  // Creates a location of type MASK from a single-channel uint8 or float
-  // cv::Mat_ (type is CV_8UC1 or CV_32FC1). Check fails if the mat is not
-  // single channel . All pixel with positive values are considered foreground,
-  // the rest background.
-  template <typename T>
-  static Location CreateCvMaskLocation(const cv::Mat_<T>& mask);
-#endif
 
   // Returns the location type describing the type of data it contains. This
   // type is set at creation time based on the one of the above factory methods.
@@ -105,14 +79,6 @@ class Location {
   // NOTE: it does not handle masks.
   Location& Scale(float scale);
 
-#if LOCATION_OPENCV
-  // Enlarges the location by the given factor. This operation keeps the center
-  // of the location fixed, while enlarging its dimensions by the given factor.
-  // Note that the location may partially lie outside the image after this
-  // operation. OpenCV required for mask enlargement. Returns *this.
-  Location& Enlarge(float factor);
-#endif
-
   // Resizes the location such that it is the tighest square location containing
   // centered the original location. It supports locations of type GLOBAL,
   // BOUNDING_BOX and RELATIVE_BOUNDING_BOX, otherwise it CHECK-fails. The user
@@ -154,12 +120,7 @@ class Location {
   T GetBBox() const;
   // Accessor for location data type RELATIVE_BOUNDING_BOX.
   Rectangle_f GetRelativeBBox() const;
-#if LOCATION_OPENCV
-  // Same as GetMask() with the difference that the return value is a cv::Mat of
-  // type CV_8UC1. It contains value 0 for background pixels and value 255 for
-  // foreground ones.
-  std::unique_ptr<cv::Mat> GetCvMask() const;
-#endif
+
   // Accessor for relative_keypoints in location data. Relative keypoints are
   // specified with x and y coordinates, where both x and y are relative to the
   // image width and height, respectively, and are in the range [0, 1]. Fails if
@@ -181,10 +142,6 @@ class Location {
   template <typename T>
   T ConvertToBBox(int image_width, int image_height) const;
   Rectangle_f ConvertToRelativeBBox(int image_width, int image_height) const;
-#if LOCATION_OPENCV
-  std::unique_ptr<cv::Mat> ConvertToCvMask(int image_width,
-                                           int image_height) const;
-#endif
   // Returns keypoints in absolute pixel coordinates.
   std::vector<Point2_i> ConvertToKeypoints(int image_width,
                                            int image_height) const;

mediapipe/framework/formats/location_opencv.cc

@@ -0,0 +1,220 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/location_opencv.h"
+
+#include "absl/memory/memory.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/formats/annotation/rasterization.pb.h"
+#include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/statusor.h"
+
+namespace mediapipe {
+
+namespace {
+Rectangle_i MaskToRectangle(const LocationData& location_data) {
+  CHECK(location_data.mask().has_rasterization());
+  const auto& rasterization = location_data.mask().rasterization();
+  if (rasterization.interval_size() == 0) {
+    return Rectangle_i(0, 0, 0, 0);
+  }
+  int xmin = std::numeric_limits<int>::max();
+  int xmax = std::numeric_limits<int>::lowest();
+  int ymin = std::numeric_limits<int>::max();
+  int ymax = std::numeric_limits<int>::lowest();
+  for (const auto& interval : rasterization.interval()) {
+    xmin = std::min(xmin, interval.left_x());
+    xmax = std::max(xmax, interval.right_x());
+    ymin = std::min(ymin, interval.y());
+    ymax = std::max(ymax, interval.y());
+  }
+  return Rectangle_i(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
+}
+
+std::unique_ptr<cv::Mat> MaskToMat(const LocationData::BinaryMask& mask) {
+  auto image = absl::make_unique<cv::Mat>();
+  *image = cv::Mat::zeros(cv::Size(mask.width(), mask.height()), CV_32FC1);
+  for (const auto& interval : mask.rasterization().interval()) {
+    for (int x = interval.left_x(); x <= interval.right_x(); ++x) {
+      image->at<float>(interval.y(), x) = 1.0f;
+    }
+  }
+  return image;
+}
+absl::StatusOr<std::unique_ptr<cv::Mat>> RectangleToMat(
+    int image_width, int image_height, const Rectangle_i& rect) {
+  // These checks prevent undefined behavior caused when setting memory for
+  // rectangles whose edges lie outside image edges.
+  if (rect.ymin() < 0 || rect.xmin() < 0 || rect.xmax() > image_width ||
+      rect.ymax() > image_height) {
+    return absl::InvalidArgumentError(absl::Substitute(
+        "Rectangle must be bounded by image boundaries.\nImage Width: "
+        "$0\nImage Height: $1\nRectangle: [($2, $3), ($4, $5)]",
+        image_width, image_height, rect.xmin(), rect.ymin(), rect.xmax(),
+        rect.ymax()));
+  }
+  // Allocate image and set pixels of foreground mask.
+  auto image = absl::make_unique<cv::Mat>();
+  *image = cv::Mat::zeros(cv::Size(image_width, image_height), CV_32FC1);
+  for (int y = rect.ymin(); y < rect.ymax(); ++y) {
+    for (int x = rect.xmin(); x < rect.xmax(); ++x) {
+      image->at<float>(y, x) = 1.0f;
+    }
+  }
+
+  return std::move(image);
+}
+}  // namespace
+
+Location CreateBBoxLocation(const cv::Rect& rect) {
+  return Location::CreateBBoxLocation(rect.x, rect.y, rect.width, rect.height);
+}
+
+std::unique_ptr<cv::Mat> GetCvMask(const Location& location) {
+  const auto location_data = location.ConvertToProto();
+  CHECK_EQ(LocationData::MASK, location_data.format());
+  const auto& mask = location_data.mask();
+  std::unique_ptr<cv::Mat> mat(
+      new cv::Mat(mask.height(), mask.width(), CV_8UC1, cv::Scalar(0)));
+  for (const auto& interval : location_data.mask().rasterization().interval()) {
+    for (int x = interval.left_x(); x <= interval.right_x(); ++x) {
+      mat->at<uint8>(interval.y(), x) = 255;
+    }
+  }
+  return mat;
+}
+
+std::unique_ptr<cv::Mat> ConvertToCvMask(const Location& location,
+                                         int image_width, int image_height) {
+  const auto location_data = location.ConvertToProto();
+  switch (location_data.format()) {
+    case LocationData::GLOBAL:
+    case LocationData::BOUNDING_BOX:
+    case LocationData::RELATIVE_BOUNDING_BOX: {
+      auto status_or_mat = RectangleToMat(
+          image_width, image_height,
+          location.ConvertToBBox<Rectangle_i>(image_width, image_height));
+      if (!status_or_mat.ok()) {
+        LOG(ERROR) << status_or_mat.status().message();
+        return nullptr;
+      }
+      return std::move(status_or_mat).value();
+    }
+    case LocationData::MASK: {
+      return MaskToMat(location_data.mask());
+    }
+  }
+// This should never happen; a new LocationData::Format enum was introduced
+// without updating this function's switch(...) to support it.
+#if !defined(MEDIAPIPE_MOBILE) && !defined(MEDIAPIPE_LITE)
+  LOG(ERROR) << "Location's LocationData has format not supported by "
+                "Location::ConvertToMask: "
+             << location_data.DebugString();
+#endif
+  return nullptr;
+}
+
+void EnlargeLocation(Location& location, const float factor) {
+  CHECK_GT(factor, 0.0f);
+  if (factor == 1.0f) return;
+  auto location_data = location.ConvertToProto();
+  switch (location_data.format()) {
+    case LocationData::GLOBAL: {
+      // Do nothing.
+      break;
+    }
+    case LocationData::BOUNDING_BOX: {
+      auto* box = location_data.mutable_bounding_box();
+      const int enlarged_int_width =
+          static_cast<int>(std::round(factor * box->width()));
+      const int enlarged_int_height =
+          static_cast<int>(std::round(factor * box->height()));
+      box->set_xmin(
+          std::max(box->xmin() + box->width() / 2 - enlarged_int_width / 2, 0));
+      box->set_ymin(std::max(
+          box->ymin() + box->height() / 2 - enlarged_int_height / 2, 0));
+      box->set_width(enlarged_int_width);
+      box->set_height(enlarged_int_height);
+      break;
+    }
+    case LocationData::RELATIVE_BOUNDING_BOX: {
+      auto* box = location_data.mutable_relative_bounding_box();
+      box->set_xmin(box->xmin() - ((factor - 1.0) * box->width()) / 2.0);
+      box->set_ymin(box->ymin() - ((factor - 1.0) * box->height()) / 2.0);
+      box->set_width(factor * box->width());
+      box->set_height(factor * box->height());
+      break;
+    }
+    case LocationData::MASK: {
+      auto mask_bounding_box = MaskToRectangle(location_data);
+      const float scaler = std::fabs(factor - 1.0f);
+      const int dilation_width =
+          static_cast<int>(std::round(scaler * mask_bounding_box.Width()));
+      const int dilation_height =
+          static_cast<int>(std::round(scaler * mask_bounding_box.Height()));
+      if (dilation_width == 0 || dilation_height == 0) break;
+      cv::Mat morph_element(dilation_height, dilation_width, CV_8U,
+                            cv::Scalar(1));
+      auto mask = GetCvMask(location);
+      if (factor > 1.0f) {
+        cv::dilate(*mask, *mask, morph_element);
+      } else {
+        cv::erode(*mask, *mask, morph_element);
+      }
+      CreateCvMaskLocation<uint8>(*mask).ConvertToProto(&location_data);
+      break;
+    }
+  }
+  location.SetFromProto(location_data);
+}
+
+template <typename T>
+Location CreateCvMaskLocation(const cv::Mat_<T>& mask) {
+  CHECK_EQ(1, mask.channels())
+      << "The specified cv::Mat mask should be single-channel.";
+
+  LocationData location_data;
+  location_data.set_format(LocationData::MASK);
+  location_data.mutable_mask()->set_width(mask.cols);
+  location_data.mutable_mask()->set_height(mask.rows);
+  auto* rasterization = location_data.mutable_mask()->mutable_rasterization();
+  const auto kForegroundThreshold = static_cast<T>(0);
+  for (int y = 0; y < mask.rows; y++) {
+    Rasterization::Interval* interval;
+    bool traversing = false;
+    for (int x = 0; x < mask.cols; x++) {
+      const bool is_foreground =
+          mask.template at<T>(y, x) > kForegroundThreshold;
+      if (is_foreground) {
+        if (!traversing) {
+          interval = rasterization->add_interval();
+          interval->set_y(y);
+          interval->set_left_x(x);
+          traversing = true;
+        }
+        interval->set_right_x(x);
+      } else {
+        traversing = false;
+      }
+    }
+  }
+  return Location(location_data);
+}
+
+template Location CreateCvMaskLocation(const cv::Mat_<uint8>& mask);
+template Location CreateCvMaskLocation(const cv::Mat_<float>& mask);
+
+}  // namespace mediapipe

mediapipe/framework/formats/location_opencv.h

@@ -0,0 +1,54 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// A collection of functions operating on MediaPipe::Location that require
+// OpenCV to either convert between formats, or apply OpenCV transformations.
+
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_LOCATION_OPENCV_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_LOCATION_OPENCV_H_
+#include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+
+namespace mediapipe {
+// Creates a location of type BOUNDING_BOX from an OpenCV rectangle.
+Location CreateBBoxLocation(const cv::Rect& rect);
+
+// Creates a location of type MASK from a single-channel uint8 or float
+// cv::Mat_ (type is CV_8UC1 or CV_32FC1). Check fails if the mat is not
+// single channel. Pixels with positive values are treated as the foreground.
+template <typename T>
+Location CreateCvMaskLocation(const cv::Mat_<T>& mask);
+
+// Enlarges the location by the given factor. This operation keeps the center
+// of the location fixed, while enlarging its dimensions by the given factor.
+// Note that the location may partially lie outside the image after this
+// operation.
+void EnlargeLocation(Location& location, float factor);
+
+// Same as Location::GetMask() with the difference that the return value is a
+// cv::Mat of type CV_8UC1. Background pixels are set to 0 and foreground pixels
+// are set to 255.
+std::unique_ptr<cv::Mat> GetCvMask(const Location& location);
+
+// Returns the provided location's RELATIVE_BOUNDING_BOX or MASK location
+// data as an OpenCV Mat. If the location data is in a format not directly
+// convertible to the specified return type the following conversion principles
+// are used:
+//   - Rectangle -> Mask: the rectangle is converted to a mask with all
+//      pixels inside the rectangle being foreground pixels.
+std::unique_ptr<cv::Mat> ConvertToCvMask(const Location& location,
+                                         int image_width, int image_height);
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_LOCATION_OPENCV_H_