Update WASM files for 0.10.2 release

PiperOrigin-RevId: 546332490

WORKSPACE

@@ -45,12 +45,13 @@ http_archive(
 )
 
 http_archive(
-   name = "rules_foreign_cc",
+    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-0.1.0",
+    sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
+    strip_prefix = "rules_foreign_cc-0.9.0",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
 )
 
-load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
 rules_foreign_cc_dependencies()
 
@@ -484,9 +485,10 @@ http_archive(
 )
 
 # TensorFlow repo should always go after the other external dependencies.
-# TF on 2023-05-26.
+# TF on 2023-06-13.
-_TENSORFLOW_GIT_COMMIT = "67d5c561981edc45daf3f9d73ddd1a77963733ca"
+_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
-_TENSORFLOW_SHA256 = "0c8326285e9cb695313e194b97d388eea70bf8bf5b13e8f0962ca8eed5179ece"
+# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
+_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
 http_archive(
     name = "org_tensorflow",
     urls = [

mediapipe/calculators/audio/BUILD

@@ -219,12 +219,10 @@ cc_library(
     deps = [
         ":time_series_framer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:integral_types",
-        "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@eigen_archive//:eigen3",
@@ -319,6 +317,20 @@ cc_test(
     ],
 )
 
+cc_binary(
+    name = "time_series_framer_calculator_benchmark",
+    srcs = ["time_series_framer_calculator_benchmark.cc"],
+    deps = [
+        ":time_series_framer_calculator",
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "@com_google_benchmark//:benchmark",
+    ],
+)
+
 cc_test(
     name = "time_series_framer_calculator_test",
     srcs = ["time_series_framer_calculator_test.cc"],

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 // Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
 const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
 
+namespace {
+std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
+    const SpectrogramCalculatorOptions::WindowType window_type) {
+  switch (window_type) {
+    // The cosine window and square root of Hann are equivalent.
+    case SpectrogramCalculatorOptions::COSINE:
+    case SpectrogramCalculatorOptions::SQRT_HANN:
+      return std::make_unique<audio_dsp::CosineWindow>();
+    case SpectrogramCalculatorOptions::HANN:
+      return std::make_unique<audio_dsp::HannWindow>();
+    case SpectrogramCalculatorOptions::HAMMING:
+      return std::make_unique<audio_dsp::HammingWindow>();
+  }
+  return nullptr;
+}
+}  // namespace
+
 absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
   SpectrogramCalculatorOptions spectrogram_options =
       cc->Options<SpectrogramCalculatorOptions>();
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
 
   output_scale_ = spectrogram_options.output_scale();
 
-  std::vector<double> window;
+  auto window_fun = MakeWindowFun(spectrogram_options.window_type());
-  switch (spectrogram_options.window_type()) {
+  if (window_fun == nullptr) {
-    case SpectrogramCalculatorOptions::COSINE:
+    return absl::Status(absl::StatusCode::kInvalidArgument,
-      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                        absl::StrCat("Invalid window type ",
-                                                   &window);
+                                     spectrogram_options.window_type()));
-      break;
-    case SpectrogramCalculatorOptions::HANN:
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      break;
-    case SpectrogramCalculatorOptions::HAMMING:
-      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                    &window);
-      break;
-    case SpectrogramCalculatorOptions::SQRT_HANN: {
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      absl::c_transform(window, window.begin(),
-                        [](double x) { return std::sqrt(x); });
-      break;
-    }
   }
+  std::vector<double> window;
+  window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
 
   // Propagate settings down to the actual Spectrogram object.
   spectrogram_generators_.clear();

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
     HANN = 0;
     HAMMING = 1;
     COSINE = 2;
-    SQRT_HANN = 4;
+    SQRT_HANN = 4;  // Alias of COSINE.
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -15,9 +15,7 @@
 // Defines TimeSeriesFramerCalculator.
 #include <math.h>
 
-#include <deque>
+#include <vector>
-#include <memory>
-#include <string>
 
 #include "Eigen/Core"
 #include "audio/dsp/window_functions.h"
@@ -25,9 +23,8 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/util/time_series_util.h"
 
 namespace mediapipe {
@@ -88,11 +85,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  // Adds input data to the internal buffer.
-  void EnqueueInput(CalculatorContext* cc);
-  // Constructs and emits framed output packets.
-  void FrameOutput(CalculatorContext* cc);
-
   Timestamp CurrentOutputTimestamp() {
     if (use_local_timestamp_) {
       return current_timestamp_;
@@ -106,14 +98,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
                  Timestamp::kTimestampUnitsPerSecond);
   }
 
-  // Returns the timestamp of a sample on a base, which is usually the time
-  // stamp of a packet.
-  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
-                                   int64_t number_of_samples) {
-    return timestamp_base + round(number_of_samples / sample_rate_ *
-                                  Timestamp::kTimestampUnitsPerSecond);
-  }
-
   // The number of input samples to advance after the current output frame is
   // emitted.
   int next_frame_step_samples() const {
@@ -142,61 +126,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   Timestamp initial_input_timestamp_;
   // The current timestamp is updated along with the incoming packets.
   Timestamp current_timestamp_;
-  int num_channels_;
 
-  // Each entry in this deque consists of a single sample, i.e. a
+  // Samples are buffered in a vector of sample blocks.
-  // single column vector, and its timestamp.
+  class SampleBlockBuffer {
-  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
+   public:
+    // Initializes the buffer.
+    void Init(double sample_rate, int num_channels) {
+      ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
+      num_channels_ = num_channels;
+      num_samples_ = 0;
+      first_block_offset_ = 0;
+    }
+
+    // Number of channels, equal to the number of rows in each Matrix.
+    int num_channels() const { return num_channels_; }
+    // Total number of available samples over all blocks.
+    int num_samples() const { return num_samples_; }
+
+    // Pushes a new block of samples on the back of the buffer with `timestamp`
+    // being the input timestamp of the packet containing the Matrix.
+    void Push(const Matrix& samples, Timestamp timestamp);
+    // Copies `count` samples from the front of the buffer. If there are fewer
+    // samples than this, the result is zero padded to have `count` samples.
+    // The timestamp of the last copied sample is written to *last_timestamp.
+    // This output is used below to update `current_timestamp_`, which is only
+    // used when `use_local_timestamp` is true.
+    Matrix CopySamples(int count, Timestamp* last_timestamp) const;
+    // Drops `count` samples from the front of the buffer. If `count` exceeds
+    // `num_samples()`, the buffer is emptied.  Returns how many samples were
+    // dropped.
+    int DropSamples(int count);
+
+   private:
+    struct Block {
+      // Matrix of num_channels rows by num_samples columns, a block of possibly
+      // multiple samples.
+      Matrix samples;
+      // Timestamp of the first sample in the Block. This comes from the input
+      // packet's timestamp that contains this Matrix.
+      Timestamp timestamp;
+
+      Block() : timestamp(Timestamp::Unstarted()) {}
+      Block(const Matrix& samples, Timestamp timestamp)
+          : samples(samples), timestamp(timestamp) {}
+      int num_samples() const { return samples.cols(); }
+    };
+    std::vector<Block> blocks_;
+    // Number of timestamp units per sample. Used to compute timestamps as
+    // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
+    double ts_units_per_sample_;
+    // Number of rows in each Matrix.
+    int num_channels_;
+    // The total number of samples over all blocks, equal to
+    // (sum_i blocks_[i].num_samples()) - first_block_offset_.
+    int num_samples_;
+    // The number of samples in the first block that have been discarded. This
+    // way we can cheaply represent "partially discarding" a block.
+    int first_block_offset_;
+  } sample_buffer_;
 
   bool use_window_;
-  Matrix window_;
+  Eigen::RowVectorXf window_;
 
   bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
 
-void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
+void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
-  const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
+                                                         Timestamp timestamp) {
-
+  num_samples_ += samples.cols();
-  for (int i = 0; i < input_frame.cols(); ++i) {
+  blocks_.emplace_back(samples, timestamp);
-    sample_buffer_.emplace_back(std::make_pair(
-        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
-  }
 }
 
-void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
+Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
-  while (sample_buffer_.size() >=
+    int count, Timestamp* last_timestamp) const {
+  Matrix copied(num_channels_, count);
+
+  if (!blocks_.empty()) {
+    int num_copied = 0;
+    // First block has an offset for samples that have been discarded.
+    int offset = first_block_offset_;
+    int n;
+    Timestamp last_block_ts;
+    int last_sample_index;
+
+    for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
+      n = std::min(it->num_samples() - offset, count);
+      // Copy `n` samples from the next block.
+      copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
+      count -= n;
+      num_copied += n;
+      last_block_ts = it->timestamp;
+      last_sample_index = offset + n - 1;
+      offset = 0;  // No samples have been discarded in subsequent blocks.
+    }
+
+    // Compute the timestamp of the last copied sample.
+    *last_timestamp =
+        last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
+  }
+
+  if (count > 0) {
+    copied.rightCols(count).setZero();  // Zero pad if needed.
+  }
+
+  return copied;
+}
+
+int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
+  if (blocks_.empty()) {
+    return 0;
+  }
+
+  auto block_it = blocks_.begin();
+  if (first_block_offset_ + count < block_it->num_samples()) {
+    // `count` is less than the remaining samples in the first block.
+    first_block_offset_ += count;
+    num_samples_ -= count;
+    return count;
+  }
+
+  int num_samples_dropped = block_it->num_samples() - first_block_offset_;
+  count -= num_samples_dropped;
+  first_block_offset_ = 0;
+
+  for (++block_it; block_it != blocks_.end(); ++block_it) {
+    if (block_it->num_samples() > count) {
+      break;
+    }
+    num_samples_dropped += block_it->num_samples();
+    count -= block_it->num_samples();
+  }
+
+  blocks_.erase(blocks_.begin(), block_it);  // Drop whole blocks.
+  if (!blocks_.empty()) {
+    first_block_offset_ = count;  // Drop part of the next block.
+    num_samples_dropped += count;
+  }
+
+  num_samples_ -= num_samples_dropped;
+  return num_samples_dropped;
+}
+
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
+  }
+
+  // Add input data to the internal buffer.
+  sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
+                      cc->InputTimestamp());
+
+  // Construct and emit framed output packets.
+  while (sample_buffer_.num_samples() >=
          frame_duration_samples_ + samples_still_to_drop_) {
-    while (samples_still_to_drop_ > 0) {
+    sample_buffer_.DropSamples(samples_still_to_drop_);
-      sample_buffer_.pop_front();
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
-      --samples_still_to_drop_;
+                                                     &current_timestamp_);
-    }
     const int frame_step_samples = next_frame_step_samples();
-    std::unique_ptr<Matrix> output_frame(
+    samples_still_to_drop_ = frame_step_samples;
-        new Matrix(num_channels_, frame_duration_samples_));
-    for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
-         ++i) {
-      output_frame->col(i) = sample_buffer_.front().first;
-      current_timestamp_ = sample_buffer_.front().second;
-      sample_buffer_.pop_front();
-    }
-    const int frame_overlap_samples =
-        frame_duration_samples_ - frame_step_samples;
-    if (frame_overlap_samples > 0) {
-      for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
-        current_timestamp_ = sample_buffer_[i].second;
-      }
-    } else {
-      samples_still_to_drop_ = -frame_overlap_samples;
-    }
 
     if (use_window_) {
-      *output_frame = (output_frame->array() * window_.array()).matrix();
+      // Apply the window to each row of output_frame.
+      output_frame.array().rowwise() *= window_.array();
     }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
-                               CurrentOutputTimestamp());
+                                         .At(CurrentOutputTimestamp()));
     ++cumulative_output_frames_;
     cumulative_completed_samples_ += frame_step_samples;
   }
@@ -206,35 +303,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
     // fact to enable packet queueing optimizations.
     cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
   }
-}
-
-absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
-  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
-    initial_input_timestamp_ = cc->InputTimestamp();
-    current_timestamp_ = initial_input_timestamp_;
-  }
-
-  EnqueueInput(cc);
-  FrameOutput(cc);
 
   return absl::OkStatus();
 }
 
 absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
-  while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
+  sample_buffer_.DropSamples(samples_still_to_drop_);
-    sample_buffer_.pop_front();
-    --samples_still_to_drop_;
-  }
-  if (!sample_buffer_.empty() && pad_final_packet_) {
-    std::unique_ptr<Matrix> output_frame(new Matrix);
-    output_frame->setZero(num_channels_, frame_duration_samples_);
-    for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i].first;
-      current_timestamp_ = sample_buffer_[i].second;
-    }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+  if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
-                               CurrentOutputTimestamp());
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+                                                     &current_timestamp_);
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
+                                         .At(CurrentOutputTimestamp()));
   }
 
   return absl::OkStatus();
@@ -258,7 +338,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
       cc->Inputs().Index(0).Header(), &input_header));
 
   sample_rate_ = input_header.sample_rate();
-  num_channels_ = input_header.num_channels();
+  sample_buffer_.Init(sample_rate_, input_header.num_channels());
   frame_duration_samples_ = time_series_util::SecondsToSamples(
       framer_options.frame_duration_seconds(), sample_rate_);
   RET_CHECK_GT(frame_duration_samples_, 0)
@@ -312,9 +392,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   }
 
   if (use_window_) {
-    window_ = Matrix::Ones(num_channels_, 1) *
+    window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
-              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
+                                             frame_duration_samples_)
-                                          frame_duration_samples_)
                   .cast<float>();
   }
   use_local_timestamp_ = framer_options.use_local_timestamp();

mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc

@@ -0,0 +1,92 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Benchmark for TimeSeriesFramerCalculator.
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/packet.h"
+
+using ::mediapipe::Matrix;
+
+void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
+  constexpr float kSampleRate = 32000.0;
+  constexpr int kNumChannels = 2;
+  constexpr int kFrameDurationSeconds = 5.0;
+  std::mt19937 rng(0 /*seed*/);
+  // Input around a half second's worth of samples at a time.
+  std::uniform_int_distribution<int> input_size_dist(15000, 17000);
+  // Generate a pool of random blocks of samples up front.
+  std::vector<Matrix> sample_pool;
+  sample_pool.reserve(20);
+  for (int i = 0; i < 20; ++i) {
+    sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
+  }
+  std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
+
+  mediapipe::CalculatorGraphConfig config;
+  config.add_input_stream("input");
+  config.add_output_stream("output");
+  auto* node = config.add_node();
+  node->set_calculator("TimeSeriesFramerCalculator");
+  node->add_input_stream("input");
+  node->add_output_stream("output");
+  mediapipe::TimeSeriesFramerCalculatorOptions* options =
+      node->mutable_options()->MutableExtension(
+          mediapipe::TimeSeriesFramerCalculatorOptions::ext);
+  options->set_frame_duration_seconds(kFrameDurationSeconds);
+
+  for (auto _ : state) {
+    state.PauseTiming();  // Pause benchmark timing.
+
+    // Prepare input packets of random blocks of samples.
+    std::vector<mediapipe::Packet> input_packets;
+    input_packets.reserve(32);
+    float t = 0;
+    for (int i = 0; i < 32; ++i) {
+      auto samples =
+          std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
+      const int num_samples = samples->cols();
+      input_packets.push_back(mediapipe::Adopt(samples.release())
+                                  .At(mediapipe::Timestamp::FromSeconds(t)));
+      t += num_samples / kSampleRate;
+    }
+    // Initialize graph.
+    mediapipe::CalculatorGraph graph;
+    CHECK_OK(graph.Initialize(config));
+    // Prepare input header.
+    auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
+    header->set_sample_rate(kSampleRate);
+    header->set_num_channels(kNumChannels);
+
+    state.ResumeTiming();  // Resume benchmark timing.
+
+    CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
+    for (auto& packet : input_packets) {
+      CHECK_OK(graph.AddPacketToInputStream("input", packet));
+    }
+    CHECK(!graph.HasError());
+    CHECK_OK(graph.CloseAllInputStreams());
+    CHECK_OK(graph.WaitUntilIdle());
+  }
+}
+BENCHMARK(BM_TimeSeriesFramerCalculator);
+
+BENCHMARK_MAIN();

mediapipe/calculators/core/BUILD

@@ -117,6 +117,7 @@ mediapipe_proto_library(
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework/formats:classification_proto",
         "//mediapipe/framework/formats:landmark_proto",
+        "//mediapipe/framework/formats:matrix_data_proto",
         "//mediapipe/framework/formats:time_series_header_proto",
     ],
 )
@@ -289,6 +290,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/api2:port",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:integral_types",
@@ -1138,6 +1140,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
@@ -1166,6 +1169,7 @@ cc_library(
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -76,4 +76,9 @@ REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
 // A calculator to process std::vector<mediapipe::Image>.
 typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
 REGISTER_CALCULATOR(BeginLoopImageCalculator);
+
+// A calculator to process std::vector<float>.
+typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
+REGISTER_CALCULATOR(BeginLoopFloatCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -104,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
     ConcatenateRenderDataVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
 
+typedef ConcatenateVectorCalculator<mediapipe::Image>
+    ConcatenateImageVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
 }  // namespace mediapipe

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
         packet.Set<LandmarkList>();
       } else if (packet_options.has_double_value()) {
         packet.Set<double>();
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set<MatrixData>();
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set<TimeSeriesHeader>();
+      } else if (packet_options.has_int64_value()) {
+        packet.Set<int64_t>();
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
             MakePacket<LandmarkList>(packet_options.landmark_list_value()));
       } else if (packet_options.has_double_value()) {
         packet.Set(MakePacket<double>(packet_options.double_value()));
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set(MakePacket<TimeSeriesHeader>(
             packet_options.time_series_header_value()));
+      } else if (packet_options.has_int64_value()) {
+        packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -19,6 +19,7 @@ package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
 import "mediapipe/framework/formats/landmark.proto";
+import "mediapipe/framework/formats/matrix_data.proto";
 import "mediapipe/framework/formats/time_series_header.proto";
 
 message ConstantSidePacketCalculatorOptions {
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
   message ConstantSidePacket {
     oneof value {
       int32 int_value = 1;
+      uint64 uint64_value = 5;
+      int64 int64_value = 11;
       float float_value = 2;
+      double double_value = 9;
       bool bool_value = 3;
       string string_value = 4;
-      uint64 uint64_value = 5;
       ClassificationList classification_list_value = 6;
       LandmarkList landmark_list_value = 7;
-      double double_value = 9;
       TimeSeriesHeader time_series_header_value = 10;
+      MatrixData matrix_data_value = 12;
     }
   }
 

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
 #include <string>
 
 #include "absl/strings/string_view.h"
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
   DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
   DoTestSingleSidePacket("{ bool_value: true }", true);
   DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+  DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
 }
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
           // However, LOOP packet is empty.
           kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
         } else {
-          kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          // Avoids sending leftovers to a stream that's already closed.
+          if (!kPrevLoop(cc).IsClosed()) {
+            kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          }
         }
         loop_packets_.pop_front();
         main_packet_specs_.pop_front();

mediapipe/calculators/core/stream_to_side_packet_calculator.cc

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // A calculator that takes a packet of an input stream and converts it to an
 // output side packet. This calculator only works under the assumption that the
@@ -28,21 +30,21 @@ namespace mediapipe {
 //   input_stream: "stream"
 //   output_side_packet: "side_packet"
 // }
-class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+class StreamToSidePacketCalculator : public Node {
  public:
-  static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
+  static constexpr Input<AnyType>::Optional kIn{""};
-    cc->Inputs().Index(0).SetAny();
+  static constexpr SideOutput<SameType<kIn>> kOut{""};
-    cc->OutputSidePackets().Index(0).SetAny();
+
-    return absl::OkStatus();
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
-  }
 
   absl::Status Process(mediapipe::CalculatorContext* cc) override {
-    mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
+    kOut(cc).Set(
-    cc->OutputSidePackets().Index(0).Set(
+        kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
-        packet.At(mediapipe::Timestamp::Unset()));
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(StreamToSidePacketCalculator);
 
+MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -135,7 +135,6 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:status",
     ],

mediapipe/calculators/image/yuv_to_image_calculator.cc

@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
   buf[0] = (fourcc >> 24) & 0xff;
   buf[1] = (fourcc >> 16) & 0xff;
   buf[2] = (fourcc >> 8) & 0xff;
-  buf[3] = (fourcc)&0xff;
+  buf[3] = (fourcc) & 0xff;
   buf[4] = 0;
   return std::string(buf);
 }

mediapipe/calculators/tensor/BUILD

@@ -228,7 +228,6 @@ cc_library(
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
@@ -280,7 +279,6 @@ cc_library(
         "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -282,18 +282,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
   if (options.has_volume_gain_db()) {
     gain_ = pow(10, options.volume_gain_db() / 20.0);
   }
-  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+  if (options.has_source_sample_rate()) {
-            !kAudioIn(cc).Header().IsEmpty())
+    source_sample_rate_ = options.source_sample_rate();
-      << "Must either specify the time series header of the \"AUDIO\" stream "
+  } else {
-         "or have the \"SAMPLE_RATE\" stream connected.";
+    RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
-  if (!kAudioIn(cc).Header().IsEmpty()) {
+              !kAudioIn(cc).Header().IsEmpty())
-    mediapipe::TimeSeriesHeader input_header;
+        << "Must either specify the time series header of the \"AUDIO\" stream "
-    MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+           "or have the \"SAMPLE_RATE\" stream connected.";
-        kAudioIn(cc).Header(), &input_header));
+    if (!kAudioIn(cc).Header().IsEmpty()) {
-    if (stream_mode_) {
+      mediapipe::TimeSeriesHeader input_header;
-      MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      MP_RETURN_IF_ERROR(
-    } else {
+          mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
-      source_sample_rate_ = input_header.sample_rate();
+              kAudioIn(cc).Header(), &input_header));
+      if (stream_mode_) {
+        MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      } else {
+        source_sample_rate_ = input_header.sample_rate();
+      }
     }
   }
   AppendZerosToSampleBuffer(padding_samples_before_);

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
   // The volume gain, measured in dB.
   // Scale the input audio amplitude by 10^(volume_gain_db/20).
   optional double volume_gain_db = 12;
+
+  // The source number of samples per second (hertz) of the input audio buffers.
+  optional double source_sample_rate = 13;
 }

mediapipe/calculators/tensor/bert_preprocessor_calculator.cc

@@ -22,7 +22,6 @@
 
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
@@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
   input_tensors.reserve(kNumInputTensorsForBert);
   for (int i = 0; i < kNumInputTensorsForBert; ++i) {
     input_tensors.push_back(
-        {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
+        {Tensor::ElementType::kInt32,
+         Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
   }
   std::memcpy(input_tensors[input_ids_tensor_index_]
                   .GetCpuWriteView()

mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc

@@ -96,6 +96,19 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
     CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
   // Read CPU input into tensors.
   RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
+
+  // If the input tensors have dynamic shape, then the tensors need to be
+  // resized and reallocated before we can copy the tensor values.
+  bool resized_tensor_shapes = false;
+  for (int i = 0; i < input_tensors.size(); ++i) {
+    if (input_tensors[i].shape().is_dynamic) {
+      interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims);
+      resized_tensor_shapes = true;
+    }
+  }
+  // Reallocation is needed for memory sanity.
+  if (resized_tensor_shapes) interpreter_->AllocateTensors();
+
   for (int i = 0; i < input_tensors.size(); ++i) {
     const TfLiteType input_tensor_type =
         interpreter_->tensor(interpreter_->inputs()[i])->type;

mediapipe/calculators/tensor/regex_preprocessor_calculator.cc

@@ -20,7 +20,6 @@
 #include <vector>
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
@@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
   // not found in the tokenizer vocab.
   std::vector<Tensor> result;
   result.push_back(
-      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
+      {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})});
   std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
               input_tokens.data(), input_tokens.size() * sizeof(int32_t));
   kTensorsOut(cc).Send(std::move(result));

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -256,6 +256,7 @@ class TensorsToDetectionsCalculator : public Node {
 
   bool gpu_inited_ = false;
   bool gpu_input_ = false;
+  bool gpu_has_enough_work_groups_ = true;
   bool anchors_init_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator);
@@ -291,7 +292,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
   auto output_detections = absl::make_unique<std::vector<Detection>>();
   bool gpu_processing = false;
-  if (CanUseGpu()) {
+  if (CanUseGpu() && gpu_has_enough_work_groups_) {
     // Use GPU processing only if at least one input tensor is already on GPU
     // (to avoid CPU->GPU overhead).
     for (const auto& tensor : *kInTensors(cc)) {
@@ -321,11 +322,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
     RET_CHECK(!has_custom_box_indices_);
   }
 
-  if (gpu_processing) {
+  if (gpu_processing && !gpu_inited_) {
-    if (!gpu_inited_) {
+    auto status = GpuInit(cc);
-      MP_RETURN_IF_ERROR(GpuInit(cc));
+    if (status.ok()) {
       gpu_inited_ = true;
+    } else if (status.code() == absl::StatusCode::kFailedPrecondition) {
+      // For initialization error because of hardware limitation, fallback to
+      // CPU processing.
+      LOG(WARNING) << status.message();
+    } else {
+      // For other error, let the error propagates.
+      return status;
     }
+  }
+  if (gpu_processing && gpu_inited_) {
     MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get()));
   } else {
     MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get()));
@@ -346,17 +356,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
     // TODO: Add flexible input tensor size handling.
     auto raw_box_tensor =
         &input_tensors[tensor_mapping_.detections_tensor_index()];
-    RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
     RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+    if (raw_box_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+      // The tensors from CPU inference has dim 3.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+    } else if (raw_box_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of box Tensor must be 3 or 4.");
+    }
     auto raw_score_tensor =
         &input_tensors[tensor_mapping_.scores_tensor_index()];
-    RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3);
+    if (raw_score_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      // The tensors from CPU inference has dim 3.
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+    } else if (raw_score_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of score Tensor must be 3 or 4.");
+    }
     auto raw_box_view = raw_box_tensor->GetCpuReadView();
     auto raw_boxes = raw_box_view.buffer<float>();
     auto raw_scores_view = raw_score_tensor->GetCpuReadView();
@@ -1111,8 +1145,13 @@ void main() {
     int max_wg_size;  //  typically <= 1024
     glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
                     &max_wg_size);  // y-dim
-    CHECK_LT(num_classes_, max_wg_size)
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
-        << "# classes must be < " << max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
     // TODO support better filtering.
     if (class_index_set_.is_allowlist) {
       CHECK_EQ(class_index_set_.values.size(),
@@ -1370,7 +1409,13 @@ kernel void scoreKernel(
         Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2});
     // # filter classes supported is hardware dependent.
     int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup;
-    CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
   }
 
 #endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)

mediapipe/calculators/tensorflow/BUILD

@@ -1077,6 +1077,7 @@ cc_test(
     linkstatic = 1,
     deps = [
         ":tensor_to_image_frame_calculator",
+        ":tensor_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:image_frame",

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc

@@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase {
 
  private:
   float scale_factor_;
+  bool scale_per_frame_min_max_;
 };
 
 REGISTER_CALCULATOR(TensorToImageFrameCalculator);
@@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
 absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) {
   scale_factor_ =
       cc->Options<TensorToImageFrameCalculatorOptions>().scale_factor();
+  scale_per_frame_min_max_ = cc->Options<TensorToImageFrameCalculatorOptions>()
+                                 .scale_per_frame_min_max();
   cc->SetOffset(TimestampDiff(0));
   return absl::OkStatus();
 }
@@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
   auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
   const int32_t total_size = height * width * depth;
 
+  if (scale_per_frame_min_max_) {
+    RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT)
+        << "Setting scale_per_frame_min_max requires FLOAT input tensors.";
+  }
   ::std::unique_ptr<const ImageFrame> output;
   if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
     // Allocate buffer with alignments.
     std::unique_ptr<uint8_t[]> buffer(
         new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
     auto data = input_tensor.flat<float>().data();
+    float min = 1e23;
+    float max = -1e23;
+    if (scale_per_frame_min_max_) {
+      for (int i = 0; i < total_size; ++i) {
+        float d = scale_factor_ * data[i];
+        if (d < min) {
+          min = d;
+        }
+        if (d > max) {
+          max = d;
+        }
+      }
+    }
     for (int i = 0; i < total_size; ++i) {
-      float d = scale_factor_ * data[i];
+      float d = data[i];
-      if (d < 0) d = 0;
+      if (scale_per_frame_min_max_) {
-      if (d > 255) d = 255;
+        d = 255 * (d - min) / (max - min + 1e-9);
+      } else {
+        d = scale_factor_ * d;
+        if (d < 0) d = 0;
+        if (d > 255) d = 255;
+      }
       buffer[i] = d;
     }
     output = ::absl::make_unique<ImageFrame>(

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto

@@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions {
   // Multiples floating point tensor outputs by this value before converting to
   // uint8. This is useful for converting from range [0, 1] to [0, 255]
   optional float scale_factor = 1 [default = 1.0];
+
+  // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255]
+  // per frame. This overrides any explicit scale_factor.
+  optional bool scale_per_frame_min_max = 2 [default = false];
 }

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc

@@ -11,7 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <type_traits>
 
+#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE";
 template <class TypeParam>
 class TensorToImageFrameCalculatorTest : public ::testing::Test {
  protected:
-  void SetUpRunner() {
+  void SetUpRunner(bool scale_per_frame_min_max = false) {
     CalculatorGraphConfig::Node config;
     config.set_calculator("TensorToImageFrameCalculator");
     config.add_input_stream("TENSOR:input_tensor");
     config.add_output_stream("IMAGE:output_image");
+    config.mutable_options()
+        ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext)
+        ->set_scale_per_frame_min_max(scale_per_frame_min_max);
     runner_ = absl::make_unique<CalculatorRunner>(config);
   }
 
@@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest,
   }
 }
 
+TYPED_TEST(TensorToImageFrameCalculatorTest,
+           Converts3DTensorToImageFrame2DGrayWithScaling) {
+  this->SetUpRunner(true);
+  auto& runner = this->runner_;
+  constexpr int kWidth = 16;
+  constexpr int kHeight = 8;
+  const tf::TensorShape tensor_shape{kHeight, kWidth};
+  auto tensor = absl::make_unique<tf::Tensor>(
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
+
+  // Writing sequence of integers as floats which we want normalized.
+  tensor_vec[0] = 255;
+  for (int i = 1; i < kWidth * kHeight; ++i) {
+    tensor_vec[i] = 200;
+  }
+
+  const int64_t time = 1234;
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  if (!std::is_same<TypeParam, float>::value) {
+    EXPECT_FALSE(runner->Run().ok());
+    return;  // Short circuit because does not apply to other types.
+  } else {
+    EXPECT_TRUE(runner->Run().ok());
+    const std::vector<Packet>& output_packets =
+        runner->Outputs().Tag(kImage).packets;
+    EXPECT_EQ(1, output_packets.size());
+    EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+    const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+    EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
+    EXPECT_EQ(kWidth, output_image.Width());
+    EXPECT_EQ(kHeight, output_image.Height());
+
+    EXPECT_EQ(255, output_image.PixelData()[0]);
+    for (int i = 1; i < kWidth * kHeight; ++i) {
+      const uint8_t pixel_value = output_image.PixelData()[i];
+      ASSERT_EQ(0, pixel_value);
+    }
+  }
+}
+
 }  // namespace mediapipe

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc

@@ -124,7 +124,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     int center_row = out_lms.landmark(lm_index).y() * hm_height;
     // Point is outside of the image let's keep it intact.
     if (center_col < 0 || center_col >= hm_width || center_row < 0 ||
-        center_col >= hm_height) {
+        center_row >= hm_height) {
       continue;
     }
 

mediapipe/examples/ios/facedetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectioncpu",

mediapipe/examples/ios/facedetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectiongpu",

mediapipe/examples/ios/faceeffect/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "faceeffect",

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facemeshgpu",

mediapipe/examples/ios/handdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handdetectiongpu",

mediapipe/examples/ios/handtrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handtrackinggpu",

mediapipe/examples/ios/helloworld/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "helloworld",

mediapipe/examples/ios/holistictrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "holistictrackinggpu",

mediapipe/examples/ios/iristrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "iristrackinggpu",

mediapipe/examples/ios/objectdetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectioncpu",

mediapipe/examples/ios/objectdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiongpu",

mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiontrackinggpu",

mediapipe/examples/ios/posetrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "posetrackinggpu",

mediapipe/examples/ios/selfiesegmentationgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "selfiesegmentationgpu",

mediapipe/framework/BUILD

@@ -1355,6 +1355,23 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "calculator_graph_summary_packet_test",
+    srcs = ["calculator_graph_summary_packet_test.cc"],
+    deps = [
+        ":calculator_framework",
+        ":packet",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_test(
     name = "calculator_runner_test",
     size = "medium",

mediapipe/framework/api2/builder.h

@@ -32,7 +32,7 @@ template <class T>
 struct dependent_false : std::false_type {};
 
 template <typename T>
-T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, int index) {
+T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, size_t index) {
   auto& vec = *vecp;
   if (vec.size() <= index) {
     vec.resize(index + 1);

mediapipe/framework/api2/node.h

@@ -88,8 +88,7 @@ struct NodeRegistrationStatic {
   static mediapipe::RegistrationToken Make() {
     return mediapipe::CalculatorBaseRegistry::Register(
         T::kCalculatorName,
-        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>,
+        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>);
-        __FILE__, __LINE__);
   }
 
   using RequireStatics = ForceStaticInstantiation<&registration>;
@@ -105,8 +104,8 @@ struct SubgraphRegistrationImpl {
   static NoDestructor<mediapipe::RegistrationToken> registration;
 
   static mediapipe::RegistrationToken Make() {
-    return mediapipe::SubgraphRegistry::Register(
+    return mediapipe::SubgraphRegistry::Register(T::kCalculatorName,
-        T::kCalculatorName, absl::make_unique<T>, __FILE__, __LINE__);
+                                                 absl::make_unique<T>);
   }
 
   using RequireStatics = ForceStaticInstantiation<&registration>;
@@ -224,13 +223,12 @@ class SubgraphImpl : public Subgraph, public Intf {
 
 // This macro is used to register a calculator that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                   \
+#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                  \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>                \
+  static mediapipe::NoDestructor<mediapipe::RegistrationToken>               \
-  REGISTRY_STATIC_VAR(calculator_registration,                                \
+  REGISTRY_STATIC_VAR(calculator_registration,                               \
-                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register(  \
+                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register( \
-      Impl::kCalculatorName,                                                  \
+      Impl::kCalculatorName,                                                 \
-      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>, \
+      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>))
-      __FILE__, __LINE__))
 
 // This macro is used to register a non-split-contract calculator. Deprecated.
 #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
@@ -241,7 +239,7 @@ class SubgraphImpl : public Subgraph, public Intf {
   static mediapipe::NoDestructor<mediapipe::RegistrationToken>         \
   REGISTRY_STATIC_VAR(subgraph_registration,                           \
                       __LINE__)(mediapipe::SubgraphRegistry::Register( \
-      Impl::kCalculatorName, absl::make_unique<Impl>, __FILE__, __LINE__))
+      Impl::kCalculatorName, absl::make_unique<Impl>))
 
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/calculator_base_test.cc

@@ -183,8 +183,7 @@ TEST(CalculatorTest, CreateByNameWhitelisted) {
   CalculatorBaseRegistry::Register(
       "::mediapipe::test_ns::whitelisted_ns::DeadCalculator",
       absl::make_unique<internal::CalculatorBaseFactoryFor<
-          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>,
+          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>);
-      __FILE__, __LINE__);
 
   // A whitelisted calculator can be found in its own namespace.
   MP_EXPECT_OK(CalculatorBaseRegistry::CreateByNameInNamespace(  //

mediapipe/framework/calculator_context.h

@@ -109,9 +109,20 @@ class CalculatorContext {
   // use OutputStream::SetOffset() directly.
   void SetOffset(TimestampDiff offset);
 
-  // Returns the status of the graph run.
+  // DEPRECATED: This was intended to get graph run status during
+  // `CalculatorBase::Close` call. However, `Close` can run simultaneously with
+  // other calculators `CalculatorBase::Process`, hence the actual graph
+  // status may change any time and returned graph status here does not
+  // necessarily reflect the actual graph status.
   //
-  // NOTE: This method should only be called during CalculatorBase::Close().
+  // As an alternative, instead of checking graph status in `Close` and doing
+  // work for "done" state, you can enable timestamp bound processing for your
+  // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger
+  // `Process` on timestamp bound updates and handle "done" state there.
+  // Check examples in:
+  // mediapipe/framework/calculator_graph_summary_packet_test.cc.
+  //
+  ABSL_DEPRECATED("Does not reflect the actual graph status.")
   absl::Status GraphStatus() const { return graph_status_; }
 
   ProfilingContext* GetProfilingContext() const {

mediapipe/framework/calculator_graph.cc

@@ -839,6 +839,13 @@ absl::Status CalculatorGraph::PrepareForRun(
 }
 
 absl::Status CalculatorGraph::WaitUntilIdle() {
+  if (has_sources_) {
+    LOG_FIRST_N(WARNING, 1)
+        << "WaitUntilIdle called on a graph with source nodes, which "
+           "is not fully supported at the moment. Source nodes: "
+        << ListSourceNodes();
+  }
+
   MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle());
   VLOG(2) << "Scheduler idle.";
   absl::Status status = absl::OkStatus();
@@ -1368,6 +1375,16 @@ const OutputStreamManager* CalculatorGraph::FindOutputStreamManager(
               .get()[validated_graph_->OutputStreamIndex(name)];
 }
 
+std::string CalculatorGraph::ListSourceNodes() const {
+  std::vector<std::string> sources;
+  for (auto& node : nodes_) {
+    if (node->IsSource()) {
+      sources.push_back(node->DebugName());
+    }
+  }
+  return absl::StrJoin(sources, ", ");
+}
+
 namespace {
 void PrintTimingToInfo(const std::string& label, int64_t timer_value) {
   const int64_t total_seconds = timer_value / 1000000ll;

mediapipe/framework/calculator_graph.h

@@ -229,8 +229,11 @@ class CalculatorGraph {
   // Wait until the running graph is in the idle mode, which is when nothing can
   // be scheduled and nothing is running in the worker threads. This function
   // can be called only after StartRun().
+  //
   // NOTE: The graph must not have any source nodes because source nodes prevent
   // the running graph from becoming idle until the source nodes are done.
+  // Currently, `WaitUntilIdle` cannot be used reliably on graphs with any
+  // source nodes.
   absl::Status WaitUntilIdle();
 
   // Wait until a packet is emitted on one of the observed output streams.
@@ -594,6 +597,9 @@ class CalculatorGraph {
   // status before taking any action.
   void UpdateThrottledNodes(InputStreamManager* stream, bool* stream_was_full);
 
+  // Returns a comma-separated list of source nodes.
+  std::string ListSourceNodes() const;
+
 #if !MEDIAPIPE_DISABLE_GPU
   // Owns the legacy GpuSharedData if we need to create one for backwards
   // compatibility.

mediapipe/framework/calculator_graph_summary_packet_test.cc

@@ -0,0 +1,430 @@
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+using ::mediapipe::api2::Input;
+using ::mediapipe::api2::Node;
+using ::mediapipe::api2::Output;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsEmpty;
+using ::testing::Value;
+
+namespace {
+
+MATCHER_P2(IntPacket, value, timestamp, "") {
+  *result_listener << "where object is (value: " << arg.template Get<int>()
+                   << ", timestamp: " << arg.Timestamp() << ")";
+  return Value(arg.template Get<int>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(timestamp));
+}
+
+// Calculates and produces sum of all passed inputs when no more packets can be
+// expected on the input stream.
+class SummaryPacketCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"SUMMARY"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    // Makes sure there are no automatic timestamp bound updates when Process
+    // is called.
+    cc->SetTimestampOffset(TimestampDiff::Unset());
+    // Currently, only ImmediateInputStreamHandler supports "done" timestamp
+    // bound update. (ImmediateInputStreamhandler handles multiple input
+    // streams differently, so, in that case, calculator adjustments may be
+    // required.)
+    // TODO: update all input stream handlers to support "done"
+    // timestamp bound update.
+    cc->SetInputStreamHandler("ImmediateInputStreamHandler");
+    // Enables processing timestamp bound updates. For this use case we are
+    // specifically interested in "done" timestamp bound update. (E.g. when
+    // all input packet sources are closed.)
+    cc->SetProcessTimestampBounds(true);
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      value_ += kIn(cc).Get();
+      value_set_ = true;
+    }
+
+    if (kOut(cc).IsClosed()) {
+      // This can happen:
+      // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g.
+      //    source calculator finished generating packets sent to kIn) and
+      //    HasNextAllowedInStream() == true (which is an often case).
+      // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still
+      //    invoke Process() with Timestamp::Max to indicate "Done" timestamp
+      //    bound update.
+      return absl::OkStatus();
+    }
+
+    // TODO: input stream holding a packet with timestamp that has
+    // no next timestamp allowed in stream should always result in
+    // InputStream::IsDone() == true.
+    if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) {
+      // `Process` may or may not be invoked for "done" timestamp bound when
+      // upstream calculator fails in `Close`. Hence, extra care is needed to
+      // identify whether the calculator needs to send output.
+      // TODO: remove when "done" timestamp bound flakiness fixed.
+      if (value_set_) {
+        // kOut(cc).Send(value_) can be used here as well, however in the case
+        // of source calculator sending inputs into kIn the resulting timestamp
+        // is not well defined (e.g. it can be the last packet timestamp or
+        // Timestamp::Max())
+        // TODO: last packet from source should always result in
+        // InputStream::IsDone() == true.
+        kOut(cc).Send(value_, Timestamp::Max());
+      }
+      kOut(cc).Close();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  int value_ = 0;
+  bool value_set_ = false;
+};
+MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnClosingAllPacketSources) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp(11));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp::Max());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPreStreamTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PreStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPostStreamTimestamp) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input'
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: 'IN:input'
+          output_stream: 'SUMMARY:output'
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class IntGeneratorCalculator : public Node {
+ public:
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return tool::StatusStop();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnSourceCalculatorCompletion) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        node {
+          calculator: "IntGeneratorCalculator"
+          output_stream: "INT:int_value"
+        }
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: "IN:int_value"
+          output_stream: "SUMMARY:output"
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+class EmitOnCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnAnotherCalculatorClosure) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "EmitOnCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInClose) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInProcessCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInProcessCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInProcess) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInProcessCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/framework/deps/registration.h

@@ -16,7 +16,6 @@
 #define MEDIAPIPE_DEPS_REGISTRATION_H_
 
 #include <algorithm>
-#include <cstdint>
 #include <functional>
 #include <string>
 #include <tuple>
@@ -162,8 +161,7 @@ class FunctionRegistry {
   FunctionRegistry(const FunctionRegistry&) = delete;
   FunctionRegistry& operator=(const FunctionRegistry&) = delete;
 
-  RegistrationToken Register(absl::string_view name, Function func,
+  RegistrationToken Register(absl::string_view name, Function func)
-                             std::string filename, uint64_t line)
       ABSL_LOCKS_EXCLUDED(lock_) {
     std::string normalized_name = GetNormalizedName(name);
     absl::WriterMutexLock lock(&lock_);
@@ -173,21 +171,10 @@ class FunctionRegistry {
     }
     if (functions_.insert(std::make_pair(normalized_name, std::move(func)))
             .second) {
-#ifndef NDEBUG
-      locations_.emplace(normalized_name,
-                         std::make_pair(std::move(filename), line));
-#endif
       return RegistrationToken(
           [this, normalized_name]() { Unregister(normalized_name); });
     }
-#ifndef NDEBUG
-    LOG(FATAL) << "Function with name " << name << " already registered."
-               << " First registration at "
-               << locations_.at(normalized_name).first << ":"
-               << locations_.at(normalized_name).second;
-#else
     LOG(FATAL) << "Function with name " << name << " already registered.";
-#endif
     return RegistrationToken([]() {});
   }
 
@@ -316,11 +303,6 @@ class FunctionRegistry {
  private:
   mutable absl::Mutex lock_;
   absl::flat_hash_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
-#ifndef NDEBUG
-  // Stores filename and line number for useful debug log.
-  absl::flat_hash_map<std::string, std::pair<std::string, uint32_t>> locations_
-      ABSL_GUARDED_BY(lock_);
-#endif
 
   // For names included in NamespaceAllowlist, strips the namespace.
   std::string GetAdjustedName(absl::string_view name) {
@@ -351,10 +333,8 @@ class GlobalFactoryRegistry {
 
  public:
   static RegistrationToken Register(absl::string_view name,
-                                    typename Functions::Function func,
+                                    typename Functions::Function func) {
-                                    std::string filename, uint64_t line) {
+    return functions()->Register(name, std::move(func));
-    return functions()->Register(name, std::move(func), std::move(filename),
-                                 line);
   }
 
   // Invokes the specified factory function and returns the result.
@@ -414,12 +394,12 @@ class GlobalFactoryRegistry {
 #define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \
   static auto* REGISTRY_STATIC_VAR(registration_##name, __LINE__) =  \
       new mediapipe::RegistrationToken(                              \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
 
 #define REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, name, ...) \
   static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                       \
       new mediapipe::RegistrationToken(                                        \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
 
 }  // namespace mediapipe
 

mediapipe/framework/formats/body_rig.proto

@@ -19,7 +19,7 @@ package mediapipe;
 // Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of
 // the joint and its visibility.
 message Joint {
-  // Joint rotation in 6D contineous representation ordered as
+  // Joint rotation in 6D continuous representation ordered as
   // [a1, b1, a2, b2, a3, b3].
   //
   // Such representation is more sutable for NN model training and can be

mediapipe/framework/formats/tensor.h

@@ -117,11 +117,18 @@ class Tensor {
     Shape() = default;
     Shape(std::initializer_list<int> dimensions) : dims(dimensions) {}
     Shape(const std::vector<int>& dimensions) : dims(dimensions) {}
+    Shape(std::initializer_list<int> dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
+    Shape(const std::vector<int>& dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
     int num_elements() const {
       return std::accumulate(dims.begin(), dims.end(), 1,
                              std::multiplies<int>());
     }
     std::vector<int> dims;
+    // The Tensor has dynamic rather than static shape so the TFLite interpreter
+    // needs to be reallocated. Only relevant for CPU.
+    bool is_dynamic = false;
   };
   // Quantization parameters corresponding to the zero_point and scale value
   // made available by TfLite quantized (uint8/int8) tensors.

mediapipe/framework/formats/tensor_test.cc

@@ -2,6 +2,7 @@
 
 #include <cstring>
 #include <string>
+#include <vector>
 
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -34,6 +35,17 @@ TEST(General, TestDataTypes) {
   EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool));
 }
 
+TEST(General, TestDynamic) {
+  Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true));
+  EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4);
+  EXPECT_TRUE(t1.shape().is_dynamic);
+
+  std::vector<int> t2_dims = {4, 3, 2, 3};
+  Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true));
+  EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3);
+  EXPECT_TRUE(t2.shape().is_dynamic);
+}
+
 TEST(Cpu, TestMemoryAllocation) {
   Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3});
   auto v1 = t1.GetCpuWriteView();

mediapipe/framework/mediapipe_cc_test.bzl

@@ -15,7 +15,7 @@ def mediapipe_cc_test(
         platforms = ["linux", "android", "ios", "wasm"],
         exclude_platforms = None,
         # ios_unit_test arguments
-        ios_minimum_os_version = "11.0",
+        ios_minimum_os_version = "12.0",
         # android_cc_test arguments
         open_gl_driver = None,
         emulator_mini_boot = True,

mediapipe/framework/packet.h

@@ -466,8 +466,7 @@ struct MessageRegistrationImpl {
 template <typename T>
 NoDestructor<mediapipe::RegistrationToken>
     MessageRegistrationImpl<T>::registration(MessageHolderRegistry::Register(
-        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder,
+        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder));
-        __FILE__, __LINE__));
 
 // For non-Message payloads, this does nothing.
 template <typename T, typename Enable = void>

mediapipe/framework/port/opencv_photo_inc.h

@@ -0,0 +1,20 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+#define MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+
+#include "third_party/OpenCV/photo.hpp"
+
+#endif  // MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_

mediapipe/framework/scheduler.cc

@@ -273,8 +273,8 @@ absl::Status Scheduler::WaitForObservedOutput() {
 // Idleness requires:
 // 1. either the graph has no source nodes or all source nodes are closed, and
 // 2. no packets are added to graph input streams.
-// For simplicity, we only allow WaitUntilIdle() to be called on a graph with
+// For simplicity, we only fully support WaitUntilIdle() to be called on a graph
-// no source nodes. (This is enforced by CalculatorGraph::WaitUntilIdle().)
+// with no source nodes.
 // The application must ensure no other threads are adding packets to graph
 // input streams while a WaitUntilIdle() call is in progress.
 absl::Status Scheduler::WaitUntilIdle() {

mediapipe/framework/subgraph.cc

@@ -64,13 +64,13 @@ GraphRegistry::GraphRegistry(
 void GraphRegistry::Register(
     const std::string& type_name,
     std::function<std::unique_ptr<Subgraph>()> factory) {
-  local_factories_.Register(type_name, factory, __FILE__, __LINE__);
+  local_factories_.Register(type_name, factory);
 }
 
 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphConfig& config) {
-  Register(type_name, [config] {
+  local_factories_.Register(type_name, [config] {
     auto result = absl::make_unique<ProtoSubgraph>(config);
     return std::unique_ptr<Subgraph>(result.release());
   });
@@ -79,7 +79,7 @@ void GraphRegistry::Register(const std::string& type_name,
 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphTemplate& templ) {
-  Register(type_name, [templ] {
+  local_factories_.Register(type_name, [templ] {
     auto result = absl::make_unique<TemplateSubgraph>(templ);
     return std::unique_ptr<Subgraph>(result.release());
   });

mediapipe/framework/timestamp.cc

@@ -131,6 +131,13 @@ Timestamp Timestamp::NextAllowedInStream() const {
   return *this + 1;
 }
 
+bool Timestamp::HasNextAllowedInStream() const {
+  if (*this >= Max() || *this == PreStream()) {
+    return false;
+  }
+  return true;
+}
+
 Timestamp Timestamp::PreviousAllowedInStream() const {
   if (*this <= Min() || *this == PostStream()) {
     // Indicates that no previous timestamps may occur.

mediapipe/framework/timestamp.h

@@ -186,6 +186,10 @@ class Timestamp {
   // CHECKs that this->IsAllowedInStream().
   Timestamp NextAllowedInStream() const;
 
+  // Returns true if there's a next timestamp in the range [Min .. Max] after
+  // this one.
+  bool HasNextAllowedInStream() const;
+
   // Returns the previous timestamp in the range [Min .. Max], or
   // Unstarted() if no Packets may preceed one with this timestamp.
   Timestamp PreviousAllowedInStream() const;

mediapipe/framework/timestamp_test.cc

@@ -125,6 +125,22 @@ TEST(TimestampTest, NextAllowedInStream) {
             Timestamp::PostStream().NextAllowedInStream());
 }
 
+TEST(TimestampTest, HasNextAllowedInStream) {
+  EXPECT_TRUE(Timestamp::Min().HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Min() + 1).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(-1000).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(0).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(1000).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 2).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 1).HasNextAllowedInStream());
+
+  EXPECT_FALSE(Timestamp::PreStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Max().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::PostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::OneOverPostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Done().HasNextAllowedInStream());
+}
+
 TEST(TimestampTest, SpecialValueDifferences) {
   {  // Lower range
     const std::vector<Timestamp> timestamps = {

mediapipe/framework/tool/BUILD

@@ -530,6 +530,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],

mediapipe/framework/tool/ios.bzl

@@ -14,7 +14,7 @@
 
 """MediaPipe Task Library Helper Rules for iOS"""
 
-MPP_TASK_MINIMUM_OS_VERSION = "11.0"
+MPP_TASK_MINIMUM_OS_VERSION = "12.0"
 
 # When the static framework is built with bazel, the all header files are moved
 # to the "Headers" directory with no header path prefixes. This auxiliary rule

mediapipe/framework/tool/mediapipe_proto.bzl

@@ -50,6 +50,7 @@ def mediapipe_proto_library_impl(
         def_cc_proto = True,
         def_py_proto = True,
         def_java_lite_proto = True,
+        def_kt_lite_proto = True,
         def_objc_proto = True,
         def_java_proto = True,
         def_jspb_proto = True,
@@ -72,6 +73,7 @@ def mediapipe_proto_library_impl(
       def_cc_proto: define the cc_proto_library target
       def_py_proto: define the py_proto_library target
       def_java_lite_proto: define the java_lite_proto_library target
+      def_kt_lite_proto: define the kt_lite_proto_library target
       def_objc_proto: define the objc_proto_library target
       def_java_proto: define the java_proto_library target
       def_jspb_proto: define the jspb_proto_library target
@@ -255,6 +257,7 @@ def mediapipe_proto_library(
         def_cc_proto = True,
         def_py_proto = True,
         def_java_lite_proto = True,
+        def_kt_lite_proto = True,
         def_portable_proto = True,  # @unused
         def_objc_proto = True,
         def_java_proto = True,
@@ -281,6 +284,7 @@ def mediapipe_proto_library(
       def_cc_proto: define the cc_proto_library target
       def_py_proto: define the py_proto_library target
       def_java_lite_proto: define the java_lite_proto_library target
+      def_kt_lite_proto: define the kt_lite_proto_library target
       def_portable_proto: ignored since portable protos are gone
       def_objc_proto: define the objc_proto_library target
       def_java_proto: define the java_proto_library target
@@ -304,6 +308,7 @@ def mediapipe_proto_library(
         def_cc_proto = def_cc_proto,
         def_py_proto = def_py_proto,
         def_java_lite_proto = def_java_lite_proto,
+        def_kt_lite_proto = def_kt_lite_proto,
         def_objc_proto = def_objc_proto,
         def_java_proto = def_java_proto,
         def_jspb_proto = def_jspb_proto,
@@ -334,6 +339,7 @@ def mediapipe_proto_library(
             def_cc_proto = def_cc_proto,
             def_py_proto = def_py_proto,
             def_java_lite_proto = def_java_lite_proto,
+            def_kt_lite_proto = def_kt_lite_proto,
             def_objc_proto = def_objc_proto,
             def_java_proto = def_java_proto,
             def_jspb_proto = def_jspb_proto,

mediapipe/framework/tool/template_parser.cc

@@ -20,6 +20,7 @@
 #include <utility>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
@@ -1430,10 +1431,10 @@ std::vector<const FieldDescriptor*> GetFields(const Message* src) {
 
 // Orders map entries in dst to match src.
 void OrderMapEntries(const Message* src, Message* dst,
-                     std::set<const Message*>* seen = nullptr) {
+                     absl::flat_hash_set<const Message*>* seen = nullptr) {
-  std::unique_ptr<std::set<const Message*>> seen_owner;
+  std::unique_ptr<absl::flat_hash_set<const Message*>> seen_owner;
   if (!seen) {
-    seen_owner = std::make_unique<std::set<const Message*>>();
+    seen_owner = std::make_unique<absl::flat_hash_set<const Message*>>();
     seen = seen_owner.get();
   }
   if (seen->count(src) > 0) {

mediapipe/gpu/BUILD

@@ -1121,7 +1121,7 @@ objc_library(
     alwayslink = 1,
 )
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 test_suite(
     name = "ios",

mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java

@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
 import javax.microedition.khronos.egl.EGLConfig;
 import javax.microedition.khronos.opengles.GL10;
 
@@ -303,7 +304,7 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
   }
 
   // Use this when the texture is not a SurfaceTexture.
-  public void setNextFrame(TextureFrame frame) {
+  public void setNextFrame(@Nullable TextureFrame frame) {
     if (surfaceTexture != null) {
       Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */);
     }

mediapipe/java/com/google/mediapipe/framework/BUILD

@@ -50,7 +50,6 @@ android_library(
         "MediaPipeRunner.java",
     ],
     visibility = [
-        "//java/com/google/android/libraries/camera/effects:__subpackages__",
         "//mediapipe/java/com/google/mediapipe:__subpackages__",
     ],
     exports = [

mediapipe/java/com/google/mediapipe/glutil/ExternalTextureRenderer.java

@@ -67,6 +67,7 @@ public class ExternalTextureRenderer {
   private float[] textureTransformMatrix = new float[16];
   private boolean flipY;
   private int rotation = Surface.ROTATION_0;
+  private boolean doExplicitCpuSync = true;
 
   /** Call this to setup the shader program before rendering. */
   public void setup() {
@@ -101,6 +102,14 @@ public class ExternalTextureRenderer {
     this.rotation = rotation;
   }
 
+  /**
+   * Configures whether the renderer should do an explicit CPU synchronization using glFinish upon
+   * each {@link #render} call. Defaults to true.
+   */
+  public void setDoExplicitCpuSync(boolean doExplicitCpuSync) {
+    this.doExplicitCpuSync = doExplicitCpuSync;
+  }
+
   /**
    * Renders the surfaceTexture to the framebuffer with optional vertical flip.
    *
@@ -150,8 +159,11 @@ public class ExternalTextureRenderer {
     GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
     ShaderUtil.checkGlError("glBindTexture");
 
-    // TODO: add sync and go back to glFlush()
+    if (doExplicitCpuSync) {
-    GLES20.glFinish();
+
+      // TODO: add sync and go back to glFlush()
+      GLES20.glFinish();
+    }
   }
 
   /**

mediapipe/model_maker/python/core/BUILD

@@ -14,7 +14,10 @@
 
 # Placeholder for internal Python strict library and test compatibility macro.
 
-package(default_visibility = ["//mediapipe:__subpackages__"])
+package(default_visibility = [
+    "//cloud/ml/applications/vision/model_garden/model_oss/mediapipe:__subpackages__",
+    "//mediapipe:__subpackages__",
+])
 
 licenses(["notice"])
 

mediapipe/model_maker/python/core/hyperparameters.py

@@ -15,9 +15,12 @@
 
 import dataclasses
 import tempfile
-
 from typing import Optional
 
+import tensorflow as tf
+
+from official.common import distribute_utils
+
 
 @dataclasses.dataclass
 class BaseHParams:
@@ -43,10 +46,10 @@ class BaseHParams:
       documentation for more details:
       https://www.tensorflow.org/api_docs/python/tf/distribute/Strategy.
     num_gpus: How many GPUs to use at each worker with the
-      DistributionStrategies API. The default is -1, which means utilize all
+      DistributionStrategies API. The default is 0.
-      available GPUs.
+    tpu: The TPU resource to be used for training. This should be either the
-    tpu: The Cloud TPU to use for training. This should be either the name used
+      name used when creating the Cloud TPU, a grpc://ip.address.of.tpu:8470
-      when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.
+      url, or an empty string if using a local TPU.
   """
 
   # Parameters for train configuration
@@ -63,5 +66,16 @@ class BaseHParams:
 
   # Parameters for hardware acceleration
   distribution_strategy: str = 'off'
-  num_gpus: int = -1  # default value of -1 means use all available GPUs
+  num_gpus: int = 0
   tpu: str = ''
+  _strategy: tf.distribute.Strategy = dataclasses.field(init=False)
+
+  def __post_init__(self):
+    self._strategy = distribute_utils.get_distribution_strategy(
+        distribution_strategy=self.distribution_strategy,
+        num_gpus=self.num_gpus,
+        tpu_address=self.tpu,
+    )
+
+  def get_strategy(self):
+    return self._strategy

mediapipe/model_maker/python/core/tasks/classifier.py

@@ -43,7 +43,7 @@ class Classifier(custom_model.CustomModel):
     self._model: tf.keras.Model = None
     self._optimizer: Union[str, tf.keras.optimizers.Optimizer] = None
     self._loss_function: Union[str, tf.keras.losses.Loss] = None
-    self._metric_function: Union[str, tf.keras.metrics.Metric] = None
+    self._metric_functions: Sequence[Union[str, tf.keras.metrics.Metric]] = None
     self._callbacks: Sequence[tf.keras.callbacks.Callback] = None
     self._hparams: hp.BaseHParams = None
     self._history: tf.keras.callbacks.History = None
@@ -92,7 +92,8 @@ class Classifier(custom_model.CustomModel):
     self._model.compile(
         optimizer=self._optimizer,
         loss=self._loss_function,
-        metrics=[self._metric_function])
+        metrics=self._metric_functions,
+    )
 
     latest_checkpoint = (
         tf.train.latest_checkpoint(checkpoint_path)

mediapipe/model_maker/python/core/utils/BUILD

@@ -80,10 +80,30 @@ py_test(
     deps = [":loss_functions"],
 )
 
+######################################################################
+# Public target of the MediaPipe Model Maker Quantization Config.
+
+# Quantization Config is used to export a quantized model. Please refer
+# to the specific task documentations such as:
+# https://developers.google.com/mediapipe/solutions/vision/image_classifier/customize
+# for usage information.
+######################################################################
+py_library(
+    name = "metrics",
+    srcs = ["metrics.py"],
+)
+
+py_test(
+    name = "metrics_test",
+    srcs = ["metrics_test.py"],
+    deps = [":metrics"],
+)
+
 py_library(
     name = "quantization",
     srcs = ["quantization.py"],
     srcs_version = "PY3",
+    visibility = ["//visibility:public"],
     deps = ["//mediapipe/model_maker/python/core/data:dataset"],
 )
 

mediapipe/model_maker/python/core/utils/metrics.py

@@ -0,0 +1,104 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Metrics utility library."""
+
+import tensorflow as tf
+
+
+def _get_binary_sparse_metric(metric: tf.metrics.Metric):
+  """Helper method to create a BinarySparse version of a tf.keras.Metric.
+
+  BinarySparse is an implementation where the update_state(y_true, y_pred) takes
+  in shapes y_true=(batch_size, 1) y_pred=(batch_size, 2). Note that this only
+  supports the binary classification case, and that class_id=0 is the negative
+  class and class_id=1 is the positive class.
+
+  Currently supported tf.metric.Metric classes
+    1. BinarySparseRecallAtPrecision
+    2. BinarySparsePrecisionAtRecall
+
+  Args:
+    metric: A tf.metric.Metric class for which we want to generate a
+      BinarySparse version of this metric.
+
+  Returns:
+    A class for the BinarySparse version of the specified tf.metrics.Metric
+  """
+
+  class BinarySparseMetric(metric):
+    """A BinarySparse wrapper class for a tf.keras.Metric.
+
+    This class has the same parameters and functions as the underlying
+    metric class. For example, the parameters for BinarySparseRecallAtPrecision
+    is the same as tf.keras.metrics.RecallAtPrecision. The only new constraint
+    is that class_id must be set to 1 (or not specified) for the Binary metric.
+    """
+
+    def __init__(self, *args, **kwargs):
+      if 'class_id' in kwargs and kwargs['class_id'] != 1:
+        raise ValueError(
+            f'Custom BinarySparseMetric for class:{metric.__name__} is '
+            'only supported for class_id=1, got class_id='
+            f'{kwargs["class_id"]} instead'
+        )
+      else:
+        kwargs['class_id'] = 1
+      super().__init__(*args, **kwargs)
+
+    def update_state(self, y_true, y_pred, sample_weight=None):
+      y_true = tf.cast(tf.reshape(y_true, [-1]), tf.int32)
+      y_true_one_hot = tf.one_hot(y_true, 2)
+      return super().update_state(
+          y_true_one_hot, y_pred, sample_weight=sample_weight
+      )
+
+  return BinarySparseMetric
+
+
+def _get_sparse_metric(metric: tf.metrics.Metric):
+  """Helper method to create a Sparse version of a tf.keras.Metric.
+
+  Sparse is an implementation where the update_state(y_true, y_pred) takes in
+  shapes y_true=(batch_size, 1) and y_pred=(batch_size, num_classes).
+
+  Currently supported tf.metrics.Metric classes:
+    1. tf.metrics.Recall
+    2. tf.metrics.Precision
+
+  Args:
+    metric: A tf.metric.Metric class for which we want to generate a Sparse
+      version of this metric.
+
+  Returns:
+    A class for the Sparse version of the specified tf.keras.Metric.
+  """
+
+  class SparseMetric(metric):
+    """A Sparse wrapper class for a tf.keras.Metric."""
+
+    def update_state(self, y_true, y_pred, sample_weight=None):
+      y_pred = tf.math.argmax(y_pred, axis=-1)
+      return super().update_state(y_true, y_pred, sample_weight=sample_weight)
+
+  return SparseMetric
+
+
+SparseRecall = _get_sparse_metric(tf.metrics.Recall)
+SparsePrecision = _get_sparse_metric(tf.metrics.Precision)
+BinarySparseRecallAtPrecision = _get_binary_sparse_metric(
+    tf.metrics.RecallAtPrecision
+)
+BinarySparsePrecisionAtRecall = _get_binary_sparse_metric(
+    tf.metrics.PrecisionAtRecall
+)

mediapipe/model_maker/python/core/utils/metrics_test.py

@@ -0,0 +1,74 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from absl.testing import parameterized
+import tensorflow as tf
+
+from mediapipe.model_maker.python.core.utils import metrics
+
+
+class SparseMetricTest(tf.test.TestCase, parameterized.TestCase):
+
+  def setUp(self):
+    super().setUp()
+    self.y_true = [0, 0, 1, 1, 0, 1]
+    self.y_pred = [
+        [0.9, 0.1],  # 0, 0 y
+        [0.8, 0.2],  # 0, 0 y
+        [0.7, 0.3],  # 0, 1 n
+        [0.6, 0.4],  # 0, 1 n
+        [0.3, 0.7],  # 1, 0 y
+        [0.3, 0.7],  # 1, 1 y
+    ]
+    self.num_classes = 3
+
+  def _assert_metric_equals(self, metric, value):
+    metric.update_state(self.y_true, self.y_pred)
+    self.assertEqual(metric.result(), value)
+
+  def test_sparse_recall(self):
+    metric = metrics.SparseRecall()
+    self._assert_metric_equals(metric, 1 / 3)
+
+  def test_sparse_precision(self):
+    metric = metrics.SparsePrecision()
+    self._assert_metric_equals(metric, 1 / 2)
+
+  def test_binary_sparse_recall_at_precision(self):
+    metric = metrics.BinarySparseRecallAtPrecision(1.0)
+    self._assert_metric_equals(metric, 0.0)  # impossible to achieve precision=1
+    metric = metrics.BinarySparseRecallAtPrecision(0.4)
+    self._assert_metric_equals(metric, 1.0)
+
+  def test_binary_sparse_precision_at_recall(self):
+    metric = metrics.BinarySparsePrecisionAtRecall(1.0)
+    self._assert_metric_equals(metric, 3 / 4)
+    metric = metrics.BinarySparsePrecisionAtRecall(0.7)
+    self._assert_metric_equals(metric, 3 / 4)
+
+  def test_binary_sparse_precision_at_recall_class_id_error(self):
+    # class_id=1 case should not error
+    _ = metrics.BinarySparsePrecisionAtRecall(1.0, class_id=1)
+    # class_id=2 case should error
+    with self.assertRaisesRegex(
+        ValueError,
+        'Custom BinarySparseMetric for class:PrecisionAtRecall is only'
+        ' supported for class_id=1, got class_id=2 instead',
+    ):
+      _ = metrics.BinarySparsePrecisionAtRecall(1.0, class_id=2)
+
+
+if __name__ == '__main__':
+  tf.test.main()

mediapipe/model_maker/python/text/text_classifier/BUILD

@@ -31,11 +31,11 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         ":dataset",
+        ":hyperparameters",
         ":model_options",
         ":model_spec",
         ":text_classifier",
         ":text_classifier_options",
-        "//mediapipe/model_maker/python/core:hyperparameters",
     ],
 )
 
@@ -45,12 +45,18 @@ py_library(
     deps = ["//mediapipe/model_maker/python/text/core:bert_model_options"],
 )
 
+py_library(
+    name = "hyperparameters",
+    srcs = ["hyperparameters.py"],
+    deps = ["//mediapipe/model_maker/python/core:hyperparameters"],
+)
+
 py_library(
     name = "model_spec",
     srcs = ["model_spec.py"],
     deps = [
+        ":hyperparameters",
         ":model_options",
-        "//mediapipe/model_maker/python/core:hyperparameters",
         "//mediapipe/model_maker/python/core/utils:file_util",
         "//mediapipe/model_maker/python/text/core:bert_model_spec",
     ],
@@ -61,9 +67,9 @@ py_test(
     srcs = ["model_spec_test.py"],
     tags = ["requires-net:external"],
     deps = [
+        ":hyperparameters",
         ":model_options",
         ":model_spec",
-        "//mediapipe/model_maker/python/core:hyperparameters",
     ],
 )
 
@@ -100,9 +106,9 @@ py_library(
     name = "text_classifier_options",
     srcs = ["text_classifier_options.py"],
     deps = [
+        ":hyperparameters",
         ":model_options",
         ":model_spec",
-        "//mediapipe/model_maker/python/core:hyperparameters",
     ],
 )
 
@@ -111,13 +117,14 @@ py_library(
     srcs = ["text_classifier.py"],
     deps = [
         ":dataset",
+        ":hyperparameters",
         ":model_options",
         ":model_spec",
         ":preprocessor",
         ":text_classifier_options",
-        "//mediapipe/model_maker/python/core:hyperparameters",
         "//mediapipe/model_maker/python/core/data:dataset",
         "//mediapipe/model_maker/python/core/tasks:classifier",
+        "//mediapipe/model_maker/python/core/utils:metrics",
         "//mediapipe/model_maker/python/core/utils:model_util",
         "//mediapipe/model_maker/python/core/utils:quantization",
         "//mediapipe/tasks/python/metadata/metadata_writers:metadata_writer",

mediapipe/model_maker/python/text/text_classifier/__init__.py

@@ -13,19 +13,23 @@
 # limitations under the License.
 """MediaPipe Public Python API for Text Classifier."""
 
-from mediapipe.model_maker.python.core import hyperparameters
 from mediapipe.model_maker.python.text.text_classifier import dataset
+from mediapipe.model_maker.python.text.text_classifier import hyperparameters
 from mediapipe.model_maker.python.text.text_classifier import model_options
 from mediapipe.model_maker.python.text.text_classifier import model_spec
 from mediapipe.model_maker.python.text.text_classifier import text_classifier
 from mediapipe.model_maker.python.text.text_classifier import text_classifier_options
 
-HParams = hyperparameters.BaseHParams
+
+AverageWordEmbeddingHParams = hyperparameters.AverageWordEmbeddingHParams
+AverageWordEmbeddingModelOptions = (
+    model_options.AverageWordEmbeddingModelOptions
+)
+BertOptimizer = hyperparameters.BertOptimizer
+BertHParams = hyperparameters.BertHParams
+BertModelOptions = model_options.BertModelOptions
 CSVParams = dataset.CSVParameters
 Dataset = dataset.Dataset
-AverageWordEmbeddingModelOptions = (
-    model_options.AverageWordEmbeddingModelOptions)
-BertModelOptions = model_options.BertModelOptions
 SupportedModels = model_spec.SupportedModels
 TextClassifier = text_classifier.TextClassifier
 TextClassifierOptions = text_classifier_options.TextClassifierOptions

mediapipe/model_maker/python/text/text_classifier/hyperparameters.py

@@ -0,0 +1,54 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Hyperparameters for training object detection models."""
+
+import dataclasses
+import enum
+from typing import Union
+
+from mediapipe.model_maker.python.core import hyperparameters as hp
+
+
+@dataclasses.dataclass
+class AverageWordEmbeddingHParams(hp.BaseHParams):
+  """The hyperparameters for an AverageWordEmbeddingClassifier."""
+
+
+@enum.unique
+class BertOptimizer(enum.Enum):
+  """Supported Optimizers for Bert Text Classifier."""
+
+  ADAMW = "adamw"
+  LAMB = "lamb"
+
+
+@dataclasses.dataclass
+class BertHParams(hp.BaseHParams):
+  """The hyperparameters for a Bert Classifier.
+
+  Attributes:
+    learning_rate: Learning rate to use for gradient descent training.
+    batch_size: Batch size for training.
+    epochs: Number of training iterations over the dataset.
+    optimizer: Optimizer to use for training. Only supported values are "adamw"
+      and "lamb".
+  """
+
+  learning_rate: float = 3e-5
+  batch_size: int = 48
+  epochs: int = 2
+  optimizer: BertOptimizer = BertOptimizer.ADAMW
+
+
+HParams = Union[BertHParams, AverageWordEmbeddingHParams]

mediapipe/model_maker/python/text/text_classifier/model_spec.py

@@ -17,13 +17,11 @@ import dataclasses
 import enum
 import functools
 
-from mediapipe.model_maker.python.core import hyperparameters as hp
 from mediapipe.model_maker.python.core.utils import file_util
 from mediapipe.model_maker.python.text.core import bert_model_spec
+from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp
 from mediapipe.model_maker.python.text.text_classifier import model_options as mo
 
-# BERT-based text classifier spec inherited from BertModelSpec
-BertClassifierSpec = bert_model_spec.BertModelSpec
 
 MOBILEBERT_TINY_FILES = file_util.DownloadedFiles(
     'text_classifier/mobilebert_tiny',
@@ -31,6 +29,12 @@ MOBILEBERT_TINY_FILES = file_util.DownloadedFiles(
     is_folder=True,
 )
 
+EXBERT_FILES = file_util.DownloadedFiles(
+    'text_classifier/exbert',
+    'https://storage.googleapis.com/mediapipe-assets/exbert.tar.gz',
+    is_folder=True,
+)
+
 
 @dataclasses.dataclass
 class AverageWordEmbeddingClassifierSpec:
@@ -43,27 +47,53 @@ class AverageWordEmbeddingClassifierSpec:
   """
 
   # `learning_rate` is unused for the average word embedding model
-  hparams: hp.BaseHParams = hp.BaseHParams(
+  hparams: hp.AverageWordEmbeddingHParams = hp.AverageWordEmbeddingHParams(
-      epochs=10, batch_size=32, learning_rate=0)
+      epochs=10, batch_size=32, learning_rate=0
+  )
   model_options: mo.AverageWordEmbeddingModelOptions = (
       mo.AverageWordEmbeddingModelOptions())
   name: str = 'AverageWordEmbedding'
 
-
 average_word_embedding_classifier_spec = functools.partial(
     AverageWordEmbeddingClassifierSpec)
 
+
+@dataclasses.dataclass
+class BertClassifierSpec(bert_model_spec.BertModelSpec):
+  """Specification for a Bert classifier model.
+
+  Only overrides the hparams attribute since the rest of the attributes are
+  inherited from the BertModelSpec.
+  """
+
+  hparams: hp.BertHParams = hp.BertHParams()
+
+
 mobilebert_classifier_spec = functools.partial(
     BertClassifierSpec,
     downloaded_files=MOBILEBERT_TINY_FILES,
-    hparams=hp.BaseHParams(
+    hparams=hp.BertHParams(
         epochs=3, batch_size=48, learning_rate=3e-5, distribution_strategy='off'
     ),
     name='MobileBert',
     tflite_input_name={
         'ids': 'serving_default_input_1:0',
-        'mask': 'serving_default_input_3:0',
         'segment_ids': 'serving_default_input_2:0',
+        'mask': 'serving_default_input_3:0',
+    },
+)
+
+exbert_classifier_spec = functools.partial(
+    BertClassifierSpec,
+    downloaded_files=EXBERT_FILES,
+    hparams=hp.BertHParams(
+        epochs=3, batch_size=48, learning_rate=3e-5, distribution_strategy='off'
+    ),
+    name='ExBert',
+    tflite_input_name={
+        'ids': 'serving_default_input_1:0',
+        'segment_ids': 'serving_default_input_2:0',
+        'mask': 'serving_default_input_3:0',
     },
 )
 
@@ -73,3 +103,4 @@ class SupportedModels(enum.Enum):
   """Predefined text classifier model specs supported by Model Maker."""
   AVERAGE_WORD_EMBEDDING_CLASSIFIER = average_word_embedding_classifier_spec
   MOBILEBERT_CLASSIFIER = mobilebert_classifier_spec
+  EXBERT_CLASSIFIER = exbert_classifier_spec

mediapipe/model_maker/python/text/text_classifier/model_spec_test.py

@@ -19,7 +19,7 @@ from unittest import mock as unittest_mock
 
 import tensorflow as tf
 
-from mediapipe.model_maker.python.core import hyperparameters as hp
+from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp
 from mediapipe.model_maker.python.text.text_classifier import model_options as classifier_model_options
 from mediapipe.model_maker.python.text.text_classifier import model_spec as ms
 
@@ -57,11 +57,13 @@ class ModelSpecTest(tf.test.TestCase):
             seq_len=128, do_fine_tuning=True, dropout_rate=0.1))
     self.assertEqual(
         model_spec_obj.hparams,
-        hp.BaseHParams(
+        hp.BertHParams(
             epochs=3,
             batch_size=48,
             learning_rate=3e-5,
-            distribution_strategy='off'))
+            distribution_strategy='off',
+        ),
+    )
 
   def test_predefined_average_word_embedding_spec(self):
     model_spec_obj = (
@@ -78,15 +80,17 @@ class ModelSpecTest(tf.test.TestCase):
             dropout_rate=0.2))
     self.assertEqual(
         model_spec_obj.hparams,
-        hp.BaseHParams(
+        hp.AverageWordEmbeddingHParams(
             epochs=10,
             batch_size=32,
             learning_rate=0,
             steps_per_epoch=None,
             shuffle=False,
             distribution_strategy='off',
-            num_gpus=-1,
+            num_gpus=0,
-            tpu=''))
+            tpu='',
+        ),
+    )
 
   def test_custom_bert_spec(self):
     custom_bert_classifier_options = (
@@ -99,7 +103,7 @@ class ModelSpecTest(tf.test.TestCase):
                      custom_bert_classifier_options)
 
   def test_custom_average_word_embedding_spec(self):
-    custom_hparams = hp.BaseHParams(
+    custom_hparams = hp.AverageWordEmbeddingHParams(
         learning_rate=0.4,
         batch_size=64,
         epochs=10,
@@ -108,7 +112,8 @@ class ModelSpecTest(tf.test.TestCase):
         export_dir='foo/bar',
         distribution_strategy='mirrored',
         num_gpus=3,
-        tpu='tpu/address')
+        tpu='tpu/address',
+    )
     custom_average_word_embedding_model_options = (
         classifier_model_options.AverageWordEmbeddingModelOptions(
             seq_len=512,

mediapipe/model_maker/python/text/text_classifier/text_classifier.py

@@ -19,14 +19,16 @@ import tempfile
 from typing import Any, Optional, Sequence, Tuple
 
 import tensorflow as tf
+from tensorflow_addons import optimizers as tfa_optimizers
 import tensorflow_hub as hub
 
-from mediapipe.model_maker.python.core import hyperparameters as hp
 from mediapipe.model_maker.python.core.data import dataset as ds
 from mediapipe.model_maker.python.core.tasks import classifier
+from mediapipe.model_maker.python.core.utils import metrics
 from mediapipe.model_maker.python.core.utils import model_util
 from mediapipe.model_maker.python.core.utils import quantization
 from mediapipe.model_maker.python.text.text_classifier import dataset as text_ds
+from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp
 from mediapipe.model_maker.python.text.text_classifier import model_options as mo
 from mediapipe.model_maker.python.text.text_classifier import model_spec as ms
 from mediapipe.model_maker.python.text.text_classifier import preprocessor
@@ -54,22 +56,26 @@ def _validate(options: text_classifier_options.TextClassifierOptions):
        ms.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER)):
     raise ValueError("Expected AVERAGE_WORD_EMBEDDING_CLASSIFIER,"
                      f" got {options.supported_model}")
-  if (isinstance(options.model_options, mo.BertModelOptions) and
+  if isinstance(options.model_options, mo.BertModelOptions) and (
-      (options.supported_model != ms.SupportedModels.MOBILEBERT_CLASSIFIER)):
+      options.supported_model != ms.SupportedModels.MOBILEBERT_CLASSIFIER
+      and options.supported_model != ms.SupportedModels.EXBERT_CLASSIFIER
+  ):
     raise ValueError(
-        f"Expected MOBILEBERT_CLASSIFIER, got {options.supported_model}")
+        "Expected a Bert Classifier(MobileBERT or EXBERT), got "
+        f"{options.supported_model}"
+    )
 
 
 class TextClassifier(classifier.Classifier):
   """API for creating and training a text classification model."""
 
-  def __init__(self, model_spec: Any, hparams: hp.BaseHParams,
+  def __init__(
-               label_names: Sequence[str]):
+      self, model_spec: Any, label_names: Sequence[str], shuffle: bool
+  ):
     super().__init__(
-        model_spec=model_spec, label_names=label_names, shuffle=hparams.shuffle)
+        model_spec=model_spec, label_names=label_names, shuffle=shuffle
+    )
     self._model_spec = model_spec
-    self._hparams = hparams
-    self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir)
     self._text_preprocessor: preprocessor.TextClassifierPreprocessor = None
 
   @classmethod
@@ -106,7 +112,10 @@ class TextClassifier(classifier.Classifier):
     if options.hparams is None:
       options.hparams = options.supported_model.value().hparams
 
-    if options.supported_model == ms.SupportedModels.MOBILEBERT_CLASSIFIER:
+    if (
+        options.supported_model == ms.SupportedModels.MOBILEBERT_CLASSIFIER
+        or options.supported_model == ms.SupportedModels.EXBERT_CLASSIFIER
+    ):
       text_classifier = (
           _BertClassifier.create_bert_classifier(train_data, validation_data,
                                                  options,
@@ -123,12 +132,24 @@ class TextClassifier(classifier.Classifier):
 
     return text_classifier
 
-  def evaluate(self, data: ds.Dataset, batch_size: int = 32) -> Any:
+  def evaluate(
+      self,
+      data: ds.Dataset,
+      batch_size: int = 32,
+      desired_precisions: Optional[Sequence[float]] = None,
+      desired_recalls: Optional[Sequence[float]] = None,
+  ) -> Any:
     """Overrides Classifier.evaluate().
 
     Args:
       data: Evaluation dataset. Must be a TextClassifier Dataset.
       batch_size: Number of samples per evaluation step.
+      desired_precisions: If specified, adds a RecallAtPrecision metric per
+        desired_precisions[i] entry which tracks the recall given the constraint
+        on precision. Only supported for binary classification.
+      desired_recalls: If specified, adds a PrecisionAtRecall metric per
+        desired_recalls[i] entry which tracks the precision given the constraint
+        on recall. Only supported for binary classification.
 
     Returns:
       The loss value and accuracy.
@@ -144,6 +165,28 @@ class TextClassifier(classifier.Classifier):
 
     processed_data = self._text_preprocessor.preprocess(data)
     dataset = processed_data.gen_tf_dataset(batch_size, is_training=False)
+
+    additional_metrics = []
+    if desired_precisions and len(data.label_names) == 2:
+      for precision in desired_precisions:
+        additional_metrics.append(
+            metrics.BinarySparseRecallAtPrecision(
+                precision, name=f"recall_at_precision_{precision}"
+            )
+        )
+    if desired_recalls and len(data.label_names) == 2:
+      for recall in desired_recalls:
+        additional_metrics.append(
+            metrics.BinarySparsePrecisionAtRecall(
+                recall, name=f"precision_at_recall_{recall}"
+            )
+        )
+    metric_functions = self._metric_functions + additional_metrics
+    self._model.compile(
+        optimizer=self._optimizer,
+        loss=self._loss_function,
+        metrics=metric_functions,
+    )
     return self._model.evaluate(dataset)
 
   def export_model(
@@ -161,9 +204,8 @@ class TextClassifier(classifier.Classifier):
         path is {self._hparams.export_dir}/{model_name}.
       quantization_config: The configuration for model quantization.
     """
-    if not tf.io.gfile.exists(self._hparams.export_dir):
-      tf.io.gfile.makedirs(self._hparams.export_dir)
     tflite_file = os.path.join(self._hparams.export_dir, model_name)
+    tf.io.gfile.makedirs(os.path.dirname(tflite_file))
     metadata_file = os.path.join(self._hparams.export_dir, "metadata.json")
 
     tflite_model = model_util.convert_to_tflite(
@@ -174,7 +216,7 @@ class TextClassifier(classifier.Classifier):
     writer = self._get_metadata_writer(tflite_model, vocab_filepath)
     tflite_model_with_metadata, metadata_json = writer.populate()
     model_util.save_tflite(tflite_model_with_metadata, tflite_file)
-    with open(metadata_file, "w") as f:
+    with tf.io.gfile.GFile(metadata_file, "w") as f:
       f.write(metadata_json)
 
   @abc.abstractmethod
@@ -191,13 +233,23 @@ class _AverageWordEmbeddingClassifier(TextClassifier):
 
   _DELIM_REGEX_PATTERN = r"[^\w\']+"
 
-  def __init__(self, model_spec: ms.AverageWordEmbeddingClassifierSpec,
+  def __init__(
-               model_options: mo.AverageWordEmbeddingModelOptions,
+      self,
-               hparams: hp.BaseHParams, label_names: Sequence[str]):
+      model_spec: ms.AverageWordEmbeddingClassifierSpec,
-    super().__init__(model_spec, hparams, label_names)
+      model_options: mo.AverageWordEmbeddingModelOptions,
+      hparams: hp.AverageWordEmbeddingHParams,
+      label_names: Sequence[str],
+  ):
+    super().__init__(model_spec, label_names, hparams.shuffle)
     self._model_options = model_options
+    self._hparams = hparams
+    self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir)
     self._loss_function = "sparse_categorical_crossentropy"
-    self._metric_function = "accuracy"
+    self._metric_functions = [
+        "accuracy",
+        metrics.SparsePrecision(name="precision", dtype=tf.float32),
+        metrics.SparseRecall(name="recall", dtype=tf.float32),
+    ]
     self._text_preprocessor: (
         preprocessor.AverageWordEmbeddingClassifierPreprocessor) = None
 
@@ -306,14 +358,26 @@ class _BertClassifier(TextClassifier):
 
   _INITIALIZER_RANGE = 0.02
 
-  def __init__(self, model_spec: ms.BertClassifierSpec,
+  def __init__(
-               model_options: mo.BertModelOptions, hparams: hp.BaseHParams,
+      self,
-               label_names: Sequence[str]):
+      model_spec: ms.BertClassifierSpec,
-    super().__init__(model_spec, hparams, label_names)
+      model_options: mo.BertModelOptions,
+      hparams: hp.BertHParams,
+      label_names: Sequence[str],
+  ):
+    super().__init__(model_spec, label_names, hparams.shuffle)
+    self._hparams = hparams
+    self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir)
     self._model_options = model_options
-    self._loss_function = tf.keras.losses.SparseCategoricalCrossentropy()
+    with self._hparams.get_strategy().scope():
-    self._metric_function = tf.keras.metrics.SparseCategoricalAccuracy(
+      self._loss_function = tf.keras.losses.SparseCategoricalCrossentropy()
-        "test_accuracy", dtype=tf.float32)
+      self._metric_functions = [
+          tf.keras.metrics.SparseCategoricalAccuracy(
+              "test_accuracy", dtype=tf.float32
+          ),
+          metrics.SparsePrecision(name="precision", dtype=tf.float32),
+          metrics.SparseRecall(name="recall", dtype=tf.float32),
+      ]
     self._text_preprocessor: preprocessor.BertClassifierPreprocessor = None
 
   @classmethod
@@ -350,8 +414,9 @@ class _BertClassifier(TextClassifier):
     """
     (processed_train_data, processed_validation_data) = (
         self._load_and_run_preprocessor(train_data, validation_data))
-    self._create_model()
+    with self._hparams.get_strategy().scope():
-    self._create_optimizer(processed_train_data)
+      self._create_model()
+      self._create_optimizer(processed_train_data)
     self._train_model(processed_train_data, processed_validation_data)
 
   def _load_and_run_preprocessor(
@@ -435,11 +500,26 @@ class _BertClassifier(TextClassifier):
           initial_learning_rate=initial_lr,
           decay_schedule_fn=lr_schedule,
           warmup_steps=warmup_steps)
-
+    if self._hparams.optimizer == hp.BertOptimizer.ADAMW:
-    self._optimizer = tf.keras.optimizers.experimental.AdamW(
+      self._optimizer = tf.keras.optimizers.experimental.AdamW(
-        lr_schedule, weight_decay=0.01, epsilon=1e-6, global_clipnorm=1.0)
+          lr_schedule, weight_decay=0.01, epsilon=1e-6, global_clipnorm=1.0
-    self._optimizer.exclude_from_weight_decay(
+      )
-        var_names=["LayerNorm", "layer_norm", "bias"])
+      self._optimizer.exclude_from_weight_decay(
+          var_names=["LayerNorm", "layer_norm", "bias"]
+      )
+    elif self._hparams.optimizer == hp.BertOptimizer.LAMB:
+      self._optimizer = tfa_optimizers.LAMB(
+          lr_schedule,
+          weight_decay_rate=0.01,
+          epsilon=1e-6,
+          exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"],
+          global_clipnorm=1.0,
+      )
+    else:
+      raise ValueError(
+          "BertHParams.optimizer must be set to ADAM or "
+          f"LAMB. Got {self._hparams.optimizer}."
+      )
 
   def _save_vocab(self, vocab_filepath: str):
     tf.io.gfile.copy(

mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py

@@ -66,14 +66,16 @@ def run(data_dir,
   quantization_config = None
   if (supported_model ==
       text_classifier.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER):
-    hparams = text_classifier.HParams(
+    hparams = text_classifier.AverageWordEmbeddingHParams(
-        epochs=10, batch_size=32, learning_rate=0, export_dir=export_dir)
+        epochs=10, batch_size=32, learning_rate=0, export_dir=export_dir
+    )
   # Warning: This takes extremely long to run on CPU
   elif (
       supported_model == text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER):
     quantization_config = quantization.QuantizationConfig.for_dynamic()
-    hparams = text_classifier.HParams(
+    hparams = text_classifier.BertHParams(
-        epochs=3, batch_size=48, learning_rate=3e-5, export_dir=export_dir)
+        epochs=3, batch_size=48, learning_rate=3e-5, export_dir=export_dir
+    )
 
   # Fine-tunes the model.
   options = text_classifier.TextClassifierOptions(

mediapipe/model_maker/python/text/text_classifier/text_classifier_options.py

@@ -16,7 +16,7 @@
 import dataclasses
 from typing import Optional
 
-from mediapipe.model_maker.python.core import hyperparameters as hp
+from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp
 from mediapipe.model_maker.python.text.text_classifier import model_options as mo
 from mediapipe.model_maker.python.text.text_classifier import model_spec as ms
 
@@ -34,5 +34,5 @@ class TextClassifierOptions:
       architecture of the `supported_model`.
   """
   supported_model: ms.SupportedModels
-  hparams: Optional[hp.BaseHParams] = None
+  hparams: Optional[hp.HParams] = None
   model_options: Optional[mo.TextClassifierModelOptions] = None

mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py

@@ -66,12 +66,14 @@ class TextClassifierTest(tf.test.TestCase):
 
   def test_create_and_train_average_word_embedding_model(self):
     train_data, validation_data = self._get_data()
-    options = (
+    options = text_classifier.TextClassifierOptions(
-        text_classifier.TextClassifierOptions(
+        supported_model=(
-            supported_model=(text_classifier.SupportedModels
+            text_classifier.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER
-                             .AVERAGE_WORD_EMBEDDING_CLASSIFIER),
+        ),
-            hparams=text_classifier.HParams(
+        hparams=text_classifier.AverageWordEmbeddingHParams(
-                epochs=1, batch_size=1, learning_rate=0)))
+            epochs=1, batch_size=1, learning_rate=0
+        ),
+    )
     average_word_embedding_classifier = (
         text_classifier.TextClassifier.create(train_data, validation_data,
                                               options))
@@ -103,12 +105,15 @@ class TextClassifierTest(tf.test.TestCase):
     options = text_classifier.TextClassifierOptions(
         supported_model=text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER,
         model_options=text_classifier.BertModelOptions(
-            do_fine_tuning=False, seq_len=2),
+            do_fine_tuning=False, seq_len=2
-        hparams=text_classifier.HParams(
+        ),
+        hparams=text_classifier.BertHParams(
             epochs=1,
             batch_size=1,
             learning_rate=3e-5,
-            distribution_strategy='off'))
+            distribution_strategy='off',
+        ),
+    )
     bert_classifier = text_classifier.TextClassifier.create(
         train_data, validation_data, options)
 

mediapipe/model_maker/python/vision/face_stylizer/BUILD

@@ -20,13 +20,6 @@ licenses(["notice"])
 
 package(default_visibility = ["//mediapipe:__subpackages__"])
 
-filegroup(
-    name = "testdata",
-    srcs = glob([
-        "testdata/**",
-    ]),
-)
-
 py_library(
     name = "constants",
     srcs = ["constants.py"],
@@ -72,18 +65,11 @@ py_library(
     name = "dataset",
     srcs = ["dataset.py"],
     deps = [
+        ":constants",
         "//mediapipe/model_maker/python/core/data:classification_dataset",
-        "//mediapipe/model_maker/python/vision/core:image_utils",
+        "//mediapipe/python:_framework_bindings",
-    ],
+        "//mediapipe/tasks/python/core:base_options",
-)
+        "//mediapipe/tasks/python/vision:face_aligner",
-
-py_test(
-    name = "dataset_test",
-    srcs = ["dataset_test.py"],
-    data = [":testdata"],
-    deps = [
-        ":dataset",
-        "//mediapipe/tasks/python/test:test_utils",
     ],
 )
 

mediapipe/model_maker/python/vision/face_stylizer/constants.py

@@ -41,5 +41,11 @@ FACE_STYLIZER_W_FILES = file_util.DownloadedFiles(
     'https://storage.googleapis.com/mediapipe-assets/face_stylizer_w_avg.npy',
 )
 
+FACE_ALIGNER_TASK_FILES = file_util.DownloadedFiles(
+    'face_stylizer/face_landmarker_v2.task',
+    'https://storage.googleapis.com/mediapipe-assets/face_landmarker_v2.task',
+    is_folder=False,
+)
+
 # Dimension of the input style vector to the decoder
 STYLE_DIM = 512

mediapipe/model_maker/python/vision/face_stylizer/dataset.py

@@ -13,13 +13,37 @@
 # limitations under the License.
 """Face stylizer dataset library."""
 
+from typing import Sequence
 import logging
 import os
 
 import tensorflow as tf
 
 from mediapipe.model_maker.python.core.data import classification_dataset
-from mediapipe.model_maker.python.vision.core import image_utils
+from mediapipe.model_maker.python.vision.face_stylizer import constants
+from mediapipe.python._framework_bindings import image as image_module
+from mediapipe.tasks.python.core import base_options as base_options_module
+from mediapipe.tasks.python.vision import face_aligner
+
+
+def _preprocess_face_dataset(
+    all_image_paths: Sequence[str],
+) -> Sequence[tf.Tensor]:
+  """Preprocess face image dataset by aligning the face."""
+  path = constants.FACE_ALIGNER_TASK_FILES.get_path()
+  base_options = base_options_module.BaseOptions(model_asset_path=path)
+  options = face_aligner.FaceAlignerOptions(base_options=base_options)
+  aligner = face_aligner.FaceAligner.create_from_options(options)
+
+  preprocessed_images = []
+  for path in all_image_paths:
+    tf.compat.v1.logging.info('Preprocess image %s', path)
+    image = image_module.Image.create_from_file(path)
+    aligned_image = aligner.align(image)
+    aligned_image_tensor = tf.convert_to_tensor(aligned_image.numpy_view())
+    preprocessed_images.append(aligned_image_tensor)
+
+  return preprocessed_images
 
 
 # TODO: Change to a unlabeled dataset if it makes sense.
@@ -58,6 +82,7 @@ class Dataset(classification_dataset.ClassificationDataset):
     ):
       raise ValueError('No images found under given directory')
 
+    image_data = _preprocess_face_dataset(all_image_paths)
     label_names = sorted(
         name
         for name in os.listdir(data_root)
@@ -73,11 +98,7 @@ class Dataset(classification_dataset.ClassificationDataset):
         for path in all_image_paths
     ]
 
-    path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)
+    image_ds = tf.data.Dataset.from_tensor_slices(image_data)
-
-    image_ds = path_ds.map(
-        image_utils.load_image, num_parallel_calls=tf.data.AUTOTUNE
-    )
 
     # Load label
     label_ds = tf.data.Dataset.from_tensor_slices(

mediapipe/model_maker/python/vision/face_stylizer/dataset_test.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import numpy as np
 import tensorflow as tf
 
+from mediapipe.model_maker.python.vision.core import image_utils
 from mediapipe.model_maker.python.vision.face_stylizer import dataset
 from mediapipe.tasks.python.test import test_utils
 
@@ -22,10 +24,10 @@ class DatasetTest(tf.test.TestCase):
 
   def setUp(self):
     super().setUp()
-    self._test_data_dirname = 'input/style'
 
   def test_from_folder(self):
-    input_data_dir = test_utils.get_test_data_path(self._test_data_dirname)
+    test_data_dirname = 'input/style'
+    input_data_dir = test_utils.get_test_data_path(test_data_dirname)
     data = dataset.Dataset.from_folder(dirname=input_data_dir)
     self.assertEqual(data.num_classes, 2)
     self.assertEqual(data.label_names, ['cartoon', 'sketch'])

mediapipe/model_maker/python/vision/face_stylizer/face_stylizer.py

@@ -14,7 +14,7 @@
 """APIs to train face stylization model."""
 
 import os
-from typing import Callable, Optional
+from typing import Any, Callable, Optional
 
 import numpy as np
 import tensorflow as tf
@@ -54,7 +54,6 @@ class FaceStylizer(object):
     self._model_spec = model_spec
     self._model_options = model_options
     self._hparams = hparams
-    # TODO: Support face alignment in image preprocessor.
     self._preprocessor = image_preprocessing.Preprocessor(
         input_shape=self._model_spec.input_image_shape,
         num_classes=1,
@@ -128,7 +127,7 @@ class FaceStylizer(object):
   def _train_model(
       self,
       train_data: classification_ds.ClassificationDataset,
-      preprocessor: Optional[Callable[..., bool]] = None,
+      preprocessor: Optional[Callable[..., Any]] = None,
   ):
     """Trains the face stylizer model.
 

mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer.py

@@ -54,7 +54,7 @@ class GestureRecognizer(classifier.Classifier):
     self._model_options = model_options
     self._hparams = hparams
     self._loss_function = loss_functions.FocalLoss(gamma=self._hparams.gamma)
-    self._metric_function = 'categorical_accuracy'
+    self._metric_functions = ['categorical_accuracy']
     self._optimizer = 'adam'
     self._callbacks = self._get_callbacks()
     self._history = None

mediapipe/model_maker/python/vision/image_classifier/image_classifier.py

@@ -59,7 +59,7 @@ class ImageClassifier(classifier.Classifier):
     self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir)
     self._loss_function = tf.keras.losses.CategoricalCrossentropy(
         label_smoothing=self._hparams.label_smoothing)
-    self._metric_function = 'accuracy'
+    self._metric_functions = ['accuracy']
     self._history = None  # Training history returned from `keras_model.fit`.
 
   @classmethod

mediapipe/model_maker/python/vision/object_detector/model.py

@@ -74,8 +74,8 @@ class ObjectDetectorModel(tf.keras.Model):
       generator_config: configs.retinanet.DetectionGenerator = configs.retinanet.DetectionGenerator(),
   ) -> configs.retinanet.RetinaNet:
     model_config = configs.retinanet.RetinaNet(
-        min_level=3,
+        min_level=self._model_spec.min_level,
-        max_level=7,
+        max_level=self._model_spec.max_level,
         num_classes=self._num_classes,
         input_size=self._model_spec.input_image_shape,
         anchor=configs.retinanet.Anchor(
@@ -101,14 +101,17 @@ class ObjectDetectorModel(tf.keras.Model):
     )
     return model_config
 
-  def _build_model(self) -> tf.keras.Model:
+  def _build_model(self, omit_l2=False) -> tf.keras.Model:
     """Builds a RetinaNet object detector model."""
     input_specs = tf.keras.layers.InputSpec(
         shape=[None] + self._model_spec.input_image_shape
     )
-    l2_regularizer = tf.keras.regularizers.l2(
+    if omit_l2:
-        self._model_options.l2_weight_decay / 2.0
+      l2_regularizer = None
-    )
+    else:
+      l2_regularizer = tf.keras.regularizers.l2(
+          self._model_options.l2_weight_decay / 2.0
+      )
     model_config = self._get_model_config()
 
     return factory.build_retinanet(input_specs, model_config, l2_regularizer)
@@ -167,7 +170,7 @@ class ObjectDetectorModel(tf.keras.Model):
 
   def convert_to_qat(self) -> None:
     """Converts the model to a QAT RetinaNet model."""
-    model = self._build_model()
+    model = self._build_model(omit_l2=True)
     dummy_input = tf.zeros([1] + self._model_spec.input_image_shape)
     model(dummy_input, training=True)
     model.set_weights(self._model.get_weights())

mediapipe/model_maker/python/vision/object_detector/model_spec.py

@@ -20,18 +20,30 @@ from typing import List
 from mediapipe.model_maker.python.core.utils import file_util
 
 
-MOBILENET_V2_FILES = file_util.DownloadedFiles(
+MOBILENET_V2_I256_FILES = file_util.DownloadedFiles(
-    'object_detector/mobilenetv2',
+    'object_detector/mobilenetv2_i256',
     'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv2_ssd_i256_ckpt.tar.gz',
     is_folder=True,
 )
 
+MOBILENET_V2_I320_FILES = file_util.DownloadedFiles(
+    'object_detector/mobilenetv2_i320',
+    'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv2_ssd_i320_ckpt.tar.gz',
+    is_folder=True,
+)
+
 MOBILENET_MULTI_AVG_FILES = file_util.DownloadedFiles(
     'object_detector/mobilenetmultiavg',
     'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv3.5_ssd_coco/mobilenetv3.5_ssd_i256_ckpt.tar.gz',
     is_folder=True,
 )
 
+MOBILENET_MULTI_AVG_I384_FILES = file_util.DownloadedFiles(
+    'object_detector/mobilenetmultiavg_i384',
+    'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv3.5_ssd_i384_ckpt.tar.gz',
+    is_folder=True,
+)
+
 
 @dataclasses.dataclass
 class ModelSpec(object):
@@ -48,30 +60,66 @@ class ModelSpec(object):
   input_image_shape: List[int]
   model_id: str
 
+  # Model Config values
+  min_level: int
+  max_level: int
 
-mobilenet_v2_spec = functools.partial(
+
+mobilenet_v2_i256_spec = functools.partial(
     ModelSpec,
-    downloaded_files=MOBILENET_V2_FILES,
+    downloaded_files=MOBILENET_V2_I256_FILES,
     checkpoint_name='ckpt-277200',
     input_image_shape=[256, 256, 3],
     model_id='MobileNetV2',
+    min_level=3,
+    max_level=7,
 )
 
-mobilenet_multi_avg_spec = functools.partial(
+mobilenet_v2_i320_spec = functools.partial(
+    ModelSpec,
+    downloaded_files=MOBILENET_V2_I320_FILES,
+    checkpoint_name='ckpt-277200',
+    input_image_shape=[320, 320, 3],
+    model_id='MobileNetV2',
+    min_level=3,
+    max_level=6,
+)
+
+mobilenet_multi_avg_i256_spec = functools.partial(
     ModelSpec,
     downloaded_files=MOBILENET_MULTI_AVG_FILES,
     checkpoint_name='ckpt-277200',
     input_image_shape=[256, 256, 3],
     model_id='MobileNetMultiAVG',
+    min_level=3,
+    max_level=7,
+)
+
+mobilenet_multi_avg_i384_spec = functools.partial(
+    ModelSpec,
+    downloaded_files=MOBILENET_MULTI_AVG_I384_FILES,
+    checkpoint_name='ckpt-277200',
+    input_image_shape=[384, 384, 3],
+    model_id='MobileNetMultiAVG',
+    min_level=3,
+    max_level=7,
 )
 
 
 @enum.unique
 class SupportedModels(enum.Enum):
-  """Predefined object detector model specs supported by Model Maker."""
+  """Predefined object detector model specs supported by Model Maker.
 
-  MOBILENET_V2 = mobilenet_v2_spec
+  Supported models include the following:
-  MOBILENET_MULTI_AVG = mobilenet_multi_avg_spec
+  - MOBILENET_V2: MobileNetV2 256x256 input
+  - MOBILENET_V2_I320: MobileNetV2 320x320 input
+  - MOBILENET_MULTI_AVG: MobileNet-MultiHW-AVG 256x256 input
+  - MOBILENET_MULTI_AVG_I384: MobileNet-MultiHW-AVG 384x384 input
+  """
+  MOBILENET_V2 = mobilenet_v2_i256_spec
+  MOBILENET_V2_I320 = mobilenet_v2_i320_spec
+  MOBILENET_MULTI_AVG = mobilenet_multi_avg_i256_spec
+  MOBILENET_MULTI_AVG_I384 = mobilenet_multi_avg_i384_spec
 
   @classmethod
   def get(cls, spec: 'SupportedModels') -> 'ModelSpec':

mediapipe/model_maker/python/vision/object_detector/object_detector.py

@@ -395,7 +395,7 @@ class ObjectDetector(classifier.Classifier):
   ) -> tf.keras.optimizers.Optimizer:
     """Creates an optimizer with learning rate schedule for regular training.
 
-    Uses Keras PiecewiseConstantDecay schedule by default.
+    Uses Keras CosineDecay schedule by default.
 
     Args:
       steps_per_epoch: Steps per epoch to calculate the step boundaries from the
@@ -404,6 +404,8 @@ class ObjectDetector(classifier.Classifier):
     Returns:
       A tf.keras.optimizer.Optimizer for model training.
     """
+    total_steps = steps_per_epoch * self._hparams.epochs
+    warmup_steps = int(total_steps * 0.1)
     init_lr = self._hparams.learning_rate * self._hparams.batch_size / 256
     decay_epochs = (
         self._hparams.cosine_decay_epochs
@@ -415,6 +417,11 @@ class ObjectDetector(classifier.Classifier):
         steps_per_epoch * decay_epochs,
         self._hparams.cosine_decay_alpha,
     )
+    learning_rate = model_util.WarmUp(
+        initial_learning_rate=init_lr,
+        decay_schedule_fn=learning_rate,
+        warmup_steps=warmup_steps,
+    )
     return tf.keras.optimizers.experimental.SGD(
         learning_rate=learning_rate, momentum=0.9
     )

mediapipe/model_maker/python/vision/object_detector/preprocessor.py

@@ -32,8 +32,8 @@ class Preprocessor(object):
     self._mean_norm = model_spec.mean_norm
     self._stddev_norm = model_spec.stddev_norm
     self._output_size = model_spec.input_image_shape[:2]
-    self._min_level = 3
+    self._min_level = model_spec.min_level
-    self._max_level = 7
+    self._max_level = model_spec.max_level
     self._num_scales = 3
     self._aspect_ratios = [0.5, 1, 2]
     self._anchor_size = 3

mediapipe/objc/DrishtiAudioUtil.h

@@ -26,10 +26,9 @@ NS_ASSUME_NONNULL_BEGIN
 
 // Converts an audio sample buffer list into a `mediapipe::Matrix`.
 // Returns an error status on failure.
-absl::StatusOr<std::unique_ptr<mediapipe::Matrix>>
+absl::StatusOr<std::unique_ptr<mediapipe::Matrix>> MediaPipeConvertAudioBufferListToAudioMatrix(
-MediaPipeConvertAudioBufferListToAudioMatrix(
+    const AudioBufferList* audioBufferList, const AudioStreamBasicDescription* streamHeader,
-    const AudioBufferList* audioBufferList,
+    CMItemCount numFrames);
-    const AudioStreamBasicDescription* streamHeader, CMItemCount numFrames);
 
 NS_ASSUME_NONNULL_END
 

mediapipe/tasks/cc/audio/core/BUILD

@@ -43,6 +43,7 @@ cc_library(
         ":base_audio_task_api",
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/tasks/cc/core:task_api_factory",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",

mediapipe/tasks/cc/audio/core/audio_task_api_factory.h

@@ -27,6 +27,7 @@ limitations under the License.
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/tasks/cc/audio/core/base_audio_task_api.h"
+#include "mediapipe/tasks/cc/core/task_api_factory.h"
 #include "tensorflow/lite/core/api/op_resolver.h"
 
 namespace mediapipe {
@@ -60,13 +61,8 @@ class AudioTaskApiFactory {
             "Task graph config should only contain one task subgraph node.",
             MediaPipeTasksStatus::kInvalidTaskGraphConfigError);
       } else {
-        if (!node.options().HasExtension(Options::ext)) {
+        MP_RETURN_IF_ERROR(
-          return CreateStatusWithPayload(
+            tasks::core::TaskApiFactory::CheckHasValidOptions<Options>(node));
-              absl::StatusCode::kInvalidArgument,
-              absl::StrCat(node.calculator(),
-                           " is missing the required task options field."),
-              MediaPipeTasksStatus::kInvalidTaskGraphConfigError);
-        }
         found_task_subgraph = true;
       }
     }

mediapipe/tasks/cc/core/BUILD

@@ -29,6 +29,7 @@ cc_library(
         "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto",
         "//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
         "//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
+        "@com_google_absl//absl/log",
         "@com_google_absl//absl/memory",
         "@org_tensorflow//tensorflow/lite/core/api:op_resolver",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",