Ensure that -std=c++14/17 is the first argument passed to Glog

PiperOrigin-RevId: 552509553
Updat WASM binaries for 0.10.3 release
2023-07-31 09:47:32 -07:00 · 2023-07-28 16:17:30 -07:00 · 2023-07-28 16:07:12 -07:00 · 2023-07-28 13:56:56 -07:00 · 2023-07-28 12:58:42 -07:00 · 2023-07-28 12:05:38 -07:00
318 changed files with 13767 additions and 1533 deletions
@@ -45,12 +45,13 @@ http_archive(
 )

 http_archive(
-   name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-0.1.0",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
+    name = "rules_foreign_cc",
+    sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
+    strip_prefix = "rules_foreign_cc-0.9.0",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
 )

-load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")

 rules_foreign_cc_dependencies()

@@ -156,22 +157,22 @@ http_archive(
 # 2020-08-21
 http_archive(
    name = "com_github_glog_glog",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
+    sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
    urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
    ],
 )
 http_archive(
    name = "com_github_glog_glog_no_gflags",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
+    sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
    build_file = "@//third_party:glog_no_gflags.BUILD",
    urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
    ],
    patches = [
-        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
+        "@//third_party:com_github_glog_glog.diff",
    ],
    patch_args = [
        "-p1",
@@ -484,9 +485,10 @@ http_archive(
 )

 # TensorFlow repo should always go after the other external dependencies.
-# TF on 2023-05-26.
-_TENSORFLOW_GIT_COMMIT = "67d5c561981edc45daf3f9d73ddd1a77963733ca"
-_TENSORFLOW_SHA256 = "0c8326285e9cb695313e194b97d388eea70bf8bf5b13e8f0962ca8eed5179ece"
+# TF on 2023-06-13.
+_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
+# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
+_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
 http_archive(
    name = "org_tensorflow",
    urls = [
@@ -68,30 +68,108 @@ config_setting(
    visibility = ["//visibility:public"],
 )

-# Note: this cannot just match "apple_platform_type": "macos" because that option
-# defaults to "macos" even when building on Linux!
-alias(
+# Generic MacOS.
+config_setting(
    name = "macos",
-    actual = select({
-        ":macos_i386": ":macos_i386",
-        ":macos_x86_64": ":macos_x86_64",
-        ":macos_arm64": ":macos_arm64",
-        "//conditions:default": ":macos_i386",  # Arbitrarily chosen from above.
-    }),
+    constraint_values = [
+        "@platforms//os:macos",
+    ],
    visibility = ["//visibility:public"],
 )

-# Note: this also matches on crosstool_top so that it does not produce ambiguous
-# selectors when used together with "android".
+# MacOS x86 64-bit.
+config_setting(
+    name = "macos_x86_64",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:x86_64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# MacOS ARM64.
+config_setting(
+    name = "macos_arm64",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:arm64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# Generic iOS.
 config_setting(
    name = "ios",
-    values = {
-        "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
-        "apple_platform_type": "ios",
-    },
+    constraint_values = [
+        "@platforms//os:ios",
+    ],
    visibility = ["//visibility:public"],
 )

+# iOS device ARM32.
+config_setting(
+    name = "ios_armv7",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64.
+config_setting(
+    name = "ios_arm64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64E.
+config_setting(
+    name = "ios_arm64e",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64e",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 32-bit.
+config_setting(
+    name = "ios_i386",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:x86_32",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 64-bit.
+config_setting(
+    name = "ios_x86_64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:x86_64",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator ARM64.
+config_setting(
+    name = "ios_sim_arm64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# Generic Apple.
 alias(
    name = "apple",
    actual = select({
@@ -102,49 +180,6 @@ alias(
    visibility = ["//visibility:public"],
 )

-config_setting(
-    name = "macos_i386",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin",
-    },
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "macos_x86_64",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin_x86_64",
-    },
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "macos_arm64",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin_arm64",
-    },
-    visibility = ["//visibility:public"],
-)
-
-[
-    config_setting(
-        name = arch,
-        values = {"cpu": arch},
-        visibility = ["//visibility:public"],
-    )
-    for arch in [
-        "ios_i386",
-        "ios_x86_64",
-        "ios_armv7",
-        "ios_arm64",
-        "ios_arm64e",
-        "ios_sim_arm64",
-    ]
-]
-
 config_setting(
    name = "windows",
    values = {"cpu": "x64_windows"},
@@ -219,12 +219,10 @@ cc_library(
    deps = [
        ":time_series_framer_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
        "//mediapipe/framework/formats:matrix",
        "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:integral_types",
-        "//mediapipe/framework/port:logging",
        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
        "//mediapipe/util:time_series_util",
        "@com_google_audio_tools//audio/dsp:window_functions",
        "@eigen_archive//:eigen3",
@@ -319,6 +317,20 @@ cc_test(
    ],
 )

+cc_binary(
+    name = "time_series_framer_calculator_benchmark",
+    srcs = ["time_series_framer_calculator_benchmark.cc"],
+    deps = [
+        ":time_series_framer_calculator",
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "@com_google_benchmark//:benchmark",
+    ],
+)
+
 cc_test(
    name = "time_series_framer_calculator_test",
    srcs = ["time_series_framer_calculator_test.cc"],
@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 // Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
 const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;

+namespace {
+std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
+    const SpectrogramCalculatorOptions::WindowType window_type) {
+  switch (window_type) {
+    // The cosine window and square root of Hann are equivalent.
+    case SpectrogramCalculatorOptions::COSINE:
+    case SpectrogramCalculatorOptions::SQRT_HANN:
+      return std::make_unique<audio_dsp::CosineWindow>();
+    case SpectrogramCalculatorOptions::HANN:
+      return std::make_unique<audio_dsp::HannWindow>();
+    case SpectrogramCalculatorOptions::HAMMING:
+      return std::make_unique<audio_dsp::HammingWindow>();
+  }
+  return nullptr;
+}
+}  // namespace
+
 absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
  SpectrogramCalculatorOptions spectrogram_options =
      cc->Options<SpectrogramCalculatorOptions>();
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {

  output_scale_ = spectrogram_options.output_scale();

-  std::vector<double> window;
-  switch (spectrogram_options.window_type()) {
-    case SpectrogramCalculatorOptions::COSINE:
-      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                   &window);
-      break;
-    case SpectrogramCalculatorOptions::HANN:
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      break;
-    case SpectrogramCalculatorOptions::HAMMING:
-      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                    &window);
-      break;
-    case SpectrogramCalculatorOptions::SQRT_HANN: {
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      absl::c_transform(window, window.begin(),
-                        [](double x) { return std::sqrt(x); });
-      break;
-    }
+  auto window_fun = MakeWindowFun(spectrogram_options.window_type());
+  if (window_fun == nullptr) {
+    return absl::Status(absl::StatusCode::kInvalidArgument,
+                        absl::StrCat("Invalid window type ",
+                                     spectrogram_options.window_type()));
  }
+  std::vector<double> window;
+  window_fun->GetPeriodicSamples(frame_duration_samples_, &window);

  // Propagate settings down to the actual Spectrogram object.
  spectrogram_generators_.clear();
@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
    HANN = 0;
    HAMMING = 1;
    COSINE = 2;
-    SQRT_HANN = 4;
+    SQRT_HANN = 4;  // Alias of COSINE.
  }
  optional WindowType window_type = 6 [default = HANN];

@@ -15,9 +15,7 @@
 // Defines TimeSeriesFramerCalculator.
 #include <math.h>

-#include <deque>
-#include <memory>
-#include <string>
+#include <vector>

 #include "Eigen/Core"
 #include "audio/dsp/window_functions.h"
@@ -25,9 +23,8 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/util/time_series_util.h"

 namespace mediapipe {
@@ -88,11 +85,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
  absl::Status Close(CalculatorContext* cc) override;

 private:
-  // Adds input data to the internal buffer.
-  void EnqueueInput(CalculatorContext* cc);
-  // Constructs and emits framed output packets.
-  void FrameOutput(CalculatorContext* cc);
-
  Timestamp CurrentOutputTimestamp() {
    if (use_local_timestamp_) {
      return current_timestamp_;
@@ -106,14 +98,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
                 Timestamp::kTimestampUnitsPerSecond);
  }

-  // Returns the timestamp of a sample on a base, which is usually the time
-  // stamp of a packet.
-  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
-                                   int64_t number_of_samples) {
-    return timestamp_base + round(number_of_samples / sample_rate_ *
-                                  Timestamp::kTimestampUnitsPerSecond);
-  }
-
  // The number of input samples to advance after the current output frame is
  // emitted.
  int next_frame_step_samples() const {
@@ -142,61 +126,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
  Timestamp initial_input_timestamp_;
  // The current timestamp is updated along with the incoming packets.
  Timestamp current_timestamp_;
-  int num_channels_;

-  // Each entry in this deque consists of a single sample, i.e. a
-  // single column vector, and its timestamp.
-  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
+  // Samples are buffered in a vector of sample blocks.
+  class SampleBlockBuffer {
+   public:
+    // Initializes the buffer.
+    void Init(double sample_rate, int num_channels) {
+      ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
+      num_channels_ = num_channels;
+      num_samples_ = 0;
+      first_block_offset_ = 0;
+    }
+
+    // Number of channels, equal to the number of rows in each Matrix.
+    int num_channels() const { return num_channels_; }
+    // Total number of available samples over all blocks.
+    int num_samples() const { return num_samples_; }
+
+    // Pushes a new block of samples on the back of the buffer with `timestamp`
+    // being the input timestamp of the packet containing the Matrix.
+    void Push(const Matrix& samples, Timestamp timestamp);
+    // Copies `count` samples from the front of the buffer. If there are fewer
+    // samples than this, the result is zero padded to have `count` samples.
+    // The timestamp of the last copied sample is written to *last_timestamp.
+    // This output is used below to update `current_timestamp_`, which is only
+    // used when `use_local_timestamp` is true.
+    Matrix CopySamples(int count, Timestamp* last_timestamp) const;
+    // Drops `count` samples from the front of the buffer. If `count` exceeds
+    // `num_samples()`, the buffer is emptied.  Returns how many samples were
+    // dropped.
+    int DropSamples(int count);
+
+   private:
+    struct Block {
+      // Matrix of num_channels rows by num_samples columns, a block of possibly
+      // multiple samples.
+      Matrix samples;
+      // Timestamp of the first sample in the Block. This comes from the input
+      // packet's timestamp that contains this Matrix.
+      Timestamp timestamp;
+
+      Block() : timestamp(Timestamp::Unstarted()) {}
+      Block(const Matrix& samples, Timestamp timestamp)
+          : samples(samples), timestamp(timestamp) {}
+      int num_samples() const { return samples.cols(); }
+    };
+    std::vector<Block> blocks_;
+    // Number of timestamp units per sample. Used to compute timestamps as
+    // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
+    double ts_units_per_sample_;
+    // Number of rows in each Matrix.
+    int num_channels_;
+    // The total number of samples over all blocks, equal to
+    // (sum_i blocks_[i].num_samples()) - first_block_offset_.
+    int num_samples_;
+    // The number of samples in the first block that have been discarded. This
+    // way we can cheaply represent "partially discarding" a block.
+    int first_block_offset_;
+  } sample_buffer_;

  bool use_window_;
-  Matrix window_;
+  Eigen::RowVectorXf window_;

  bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);

-void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
-  const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
-
-  for (int i = 0; i < input_frame.cols(); ++i) {
-    sample_buffer_.emplace_back(std::make_pair(
-        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
-  }
+void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
+                                                         Timestamp timestamp) {
+  num_samples_ += samples.cols();
+  blocks_.emplace_back(samples, timestamp);
 }

-void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
-  while (sample_buffer_.size() >=
+Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
+    int count, Timestamp* last_timestamp) const {
+  Matrix copied(num_channels_, count);
+
+  if (!blocks_.empty()) {
+    int num_copied = 0;
+    // First block has an offset for samples that have been discarded.
+    int offset = first_block_offset_;
+    int n;
+    Timestamp last_block_ts;
+    int last_sample_index;
+
+    for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
+      n = std::min(it->num_samples() - offset, count);
+      // Copy `n` samples from the next block.
+      copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
+      count -= n;
+      num_copied += n;
+      last_block_ts = it->timestamp;
+      last_sample_index = offset + n - 1;
+      offset = 0;  // No samples have been discarded in subsequent blocks.
+    }
+
+    // Compute the timestamp of the last copied sample.
+    *last_timestamp =
+        last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
+  }
+
+  if (count > 0) {
+    copied.rightCols(count).setZero();  // Zero pad if needed.
+  }
+
+  return copied;
+}
+
+int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
+  if (blocks_.empty()) {
+    return 0;
+  }
+
+  auto block_it = blocks_.begin();
+  if (first_block_offset_ + count < block_it->num_samples()) {
+    // `count` is less than the remaining samples in the first block.
+    first_block_offset_ += count;
+    num_samples_ -= count;
+    return count;
+  }
+
+  int num_samples_dropped = block_it->num_samples() - first_block_offset_;
+  count -= num_samples_dropped;
+  first_block_offset_ = 0;
+
+  for (++block_it; block_it != blocks_.end(); ++block_it) {
+    if (block_it->num_samples() > count) {
+      break;
+    }
+    num_samples_dropped += block_it->num_samples();
+    count -= block_it->num_samples();
+  }
+
+  blocks_.erase(blocks_.begin(), block_it);  // Drop whole blocks.
+  if (!blocks_.empty()) {
+    first_block_offset_ = count;  // Drop part of the next block.
+    num_samples_dropped += count;
+  }
+
+  num_samples_ -= num_samples_dropped;
+  return num_samples_dropped;
+}
+
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
+  }
+
+  // Add input data to the internal buffer.
+  sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
+                      cc->InputTimestamp());
+
+  // Construct and emit framed output packets.
+  while (sample_buffer_.num_samples() >=
         frame_duration_samples_ + samples_still_to_drop_) {
-    while (samples_still_to_drop_ > 0) {
-      sample_buffer_.pop_front();
-      --samples_still_to_drop_;
-    }
+    sample_buffer_.DropSamples(samples_still_to_drop_);
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+                                                     &current_timestamp_);
    const int frame_step_samples = next_frame_step_samples();
-    std::unique_ptr<Matrix> output_frame(
-        new Matrix(num_channels_, frame_duration_samples_));
-    for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
-         ++i) {
-      output_frame->col(i) = sample_buffer_.front().first;
-      current_timestamp_ = sample_buffer_.front().second;
-      sample_buffer_.pop_front();
-    }
-    const int frame_overlap_samples =
-        frame_duration_samples_ - frame_step_samples;
-    if (frame_overlap_samples > 0) {
-      for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
-        current_timestamp_ = sample_buffer_[i].second;
-      }
-    } else {
-      samples_still_to_drop_ = -frame_overlap_samples;
-    }
+    samples_still_to_drop_ = frame_step_samples;

    if (use_window_) {
-      *output_frame = (output_frame->array() * window_.array()).matrix();
+      // Apply the window to each row of output_frame.
+      output_frame.array().rowwise() *= window_.array();
    }

-    cc->Outputs().Index(0).Add(output_frame.release(),
-                               CurrentOutputTimestamp());
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
+                                         .At(CurrentOutputTimestamp()));
    ++cumulative_output_frames_;
    cumulative_completed_samples_ += frame_step_samples;
  }
@@ -206,35 +303,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
    // fact to enable packet queueing optimizations.
    cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
  }
-}
-
-absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
-  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
-    initial_input_timestamp_ = cc->InputTimestamp();
-    current_timestamp_ = initial_input_timestamp_;
-  }
-
-  EnqueueInput(cc);
-  FrameOutput(cc);

  return absl::OkStatus();
 }

 absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
-  while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
-    sample_buffer_.pop_front();
-    --samples_still_to_drop_;
-  }
-  if (!sample_buffer_.empty() && pad_final_packet_) {
-    std::unique_ptr<Matrix> output_frame(new Matrix);
-    output_frame->setZero(num_channels_, frame_duration_samples_);
-    for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i].first;
-      current_timestamp_ = sample_buffer_[i].second;
-    }
+  sample_buffer_.DropSamples(samples_still_to_drop_);

-    cc->Outputs().Index(0).Add(output_frame.release(),
-                               CurrentOutputTimestamp());
+  if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+                                                     &current_timestamp_);
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
+                                         .At(CurrentOutputTimestamp()));
  }

  return absl::OkStatus();
@@ -258,7 +338,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
      cc->Inputs().Index(0).Header(), &input_header));

  sample_rate_ = input_header.sample_rate();
-  num_channels_ = input_header.num_channels();
+  sample_buffer_.Init(sample_rate_, input_header.num_channels());
  frame_duration_samples_ = time_series_util::SecondsToSamples(
      framer_options.frame_duration_seconds(), sample_rate_);
  RET_CHECK_GT(frame_duration_samples_, 0)
@@ -312,9 +392,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
  }

  if (use_window_) {
-    window_ = Matrix::Ones(num_channels_, 1) *
-              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
-                                          frame_duration_samples_)
+    window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
+                                             frame_duration_samples_)
                  .cast<float>();
  }
  use_local_timestamp_ = framer_options.use_local_timestamp();
@@ -0,0 +1,92 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Benchmark for TimeSeriesFramerCalculator.
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/packet.h"
+
+using ::mediapipe::Matrix;
+
+void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
+  constexpr float kSampleRate = 32000.0;
+  constexpr int kNumChannels = 2;
+  constexpr int kFrameDurationSeconds = 5.0;
+  std::mt19937 rng(0 /*seed*/);
+  // Input around a half second's worth of samples at a time.
+  std::uniform_int_distribution<int> input_size_dist(15000, 17000);
+  // Generate a pool of random blocks of samples up front.
+  std::vector<Matrix> sample_pool;
+  sample_pool.reserve(20);
+  for (int i = 0; i < 20; ++i) {
+    sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
+  }
+  std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
+
+  mediapipe::CalculatorGraphConfig config;
+  config.add_input_stream("input");
+  config.add_output_stream("output");
+  auto* node = config.add_node();
+  node->set_calculator("TimeSeriesFramerCalculator");
+  node->add_input_stream("input");
+  node->add_output_stream("output");
+  mediapipe::TimeSeriesFramerCalculatorOptions* options =
+      node->mutable_options()->MutableExtension(
+          mediapipe::TimeSeriesFramerCalculatorOptions::ext);
+  options->set_frame_duration_seconds(kFrameDurationSeconds);
+
+  for (auto _ : state) {
+    state.PauseTiming();  // Pause benchmark timing.
+
+    // Prepare input packets of random blocks of samples.
+    std::vector<mediapipe::Packet> input_packets;
+    input_packets.reserve(32);
+    float t = 0;
+    for (int i = 0; i < 32; ++i) {
+      auto samples =
+          std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
+      const int num_samples = samples->cols();
+      input_packets.push_back(mediapipe::Adopt(samples.release())
+                                  .At(mediapipe::Timestamp::FromSeconds(t)));
+      t += num_samples / kSampleRate;
+    }
+    // Initialize graph.
+    mediapipe::CalculatorGraph graph;
+    CHECK_OK(graph.Initialize(config));
+    // Prepare input header.
+    auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
+    header->set_sample_rate(kSampleRate);
+    header->set_num_channels(kNumChannels);
+
+    state.ResumeTiming();  // Resume benchmark timing.
+
+    CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
+    for (auto& packet : input_packets) {
+      CHECK_OK(graph.AddPacketToInputStream("input", packet));
+    }
+    CHECK(!graph.HasError());
+    CHECK_OK(graph.CloseAllInputStreams());
+    CHECK_OK(graph.WaitUntilIdle());
+  }
+}
+BENCHMARK(BM_TimeSeriesFramerCalculator);
+
+BENCHMARK_MAIN();
@@ -117,6 +117,7 @@ mediapipe_proto_library(
        "//mediapipe/framework:calculator_proto",
        "//mediapipe/framework/formats:classification_proto",
        "//mediapipe/framework/formats:landmark_proto",
+        "//mediapipe/framework/formats:matrix_data_proto",
        "//mediapipe/framework/formats:time_series_header_proto",
    ],
 )
@@ -289,6 +290,7 @@ cc_library(
        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/api2:port",
        "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:image",
        "//mediapipe/framework/formats:landmark_cc_proto",
        "//mediapipe/framework/formats:tensor",
        "//mediapipe/framework/port:integral_types",
@@ -379,17 +381,6 @@ cc_library(
    alwayslink = 1,
 )

-cc_library(
-    name = "clip_detection_vector_size_calculator",
-    srcs = ["clip_detection_vector_size_calculator.cc"],
-    deps = [
-        ":clip_vector_size_calculator",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:detection_cc_proto",
-    ],
-    alwayslink = 1,
-)
-
 cc_test(
    name = "clip_vector_size_calculator_test",
    srcs = ["clip_vector_size_calculator_test.cc"],
@@ -1138,6 +1129,7 @@ cc_library(
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
        "//mediapipe/framework/port:status",
    ],
    alwayslink = 1,
@@ -1166,6 +1158,7 @@ cc_library(
        "//mediapipe/framework:collection_item_id",
        "//mediapipe/framework/formats:classification_cc_proto",
        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
        "//mediapipe/framework/formats:time_series_header_cc_proto",
        "//mediapipe/framework/port:integral_types",
        "//mediapipe/framework/port:ret_check",
@@ -76,4 +76,9 @@ REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
 // A calculator to process std::vector<mediapipe::Image>.
 typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
 REGISTER_CALCULATOR(BeginLoopImageCalculator);
+
+// A calculator to process std::vector<float>.
+typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
+REGISTER_CALCULATOR(BeginLoopFloatCalculator);
+
 }  // namespace mediapipe
@@ -17,6 +17,7 @@
 #include <vector>

 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -104,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
    ConcatenateRenderDataVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);

+typedef ConcatenateVectorCalculator<mediapipe::Image>
+    ConcatenateImageVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
 }  // namespace mediapipe
@@ -19,6 +19,7 @@
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
        packet.Set<LandmarkList>();
      } else if (packet_options.has_double_value()) {
        packet.Set<double>();
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set<MatrixData>();
      } else if (packet_options.has_time_series_header_value()) {
        packet.Set<TimeSeriesHeader>();
+      } else if (packet_options.has_int64_value()) {
+        packet.Set<int64_t>();
      } else {
        return absl::InvalidArgumentError(
            "None of supported values were specified in options.");
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
            MakePacket<LandmarkList>(packet_options.landmark_list_value()));
      } else if (packet_options.has_double_value()) {
        packet.Set(MakePacket<double>(packet_options.double_value()));
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
      } else if (packet_options.has_time_series_header_value()) {
        packet.Set(MakePacket<TimeSeriesHeader>(
            packet_options.time_series_header_value()));
+      } else if (packet_options.has_int64_value()) {
+        packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
      } else {
        return absl::InvalidArgumentError(
            "None of supported values were specified in options.");
@@ -19,6 +19,7 @@ package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
 import "mediapipe/framework/formats/landmark.proto";
+import "mediapipe/framework/formats/matrix_data.proto";
 import "mediapipe/framework/formats/time_series_header.proto";

 message ConstantSidePacketCalculatorOptions {
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
  message ConstantSidePacket {
    oneof value {
      int32 int_value = 1;
+      uint64 uint64_value = 5;
+      int64 int64_value = 11;
      float float_value = 2;
+      double double_value = 9;
      bool bool_value = 3;
      string string_value = 4;
-      uint64 uint64_value = 5;
      ClassificationList classification_list_value = 6;
      LandmarkList landmark_list_value = 7;
-      double double_value = 9;
      TimeSeriesHeader time_series_header_value = 10;
+      MatrixData matrix_data_value = 12;
    }
  }

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#include <cstdint>
 #include <string>

 #include "absl/strings/string_view.h"
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
  DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
  DoTestSingleSidePacket("{ bool_value: true }", true);
  DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+  DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
 }

 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
          // However, LOOP packet is empty.
          kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
        } else {
-          kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          // Avoids sending leftovers to a stream that's already closed.
+          if (!kPrevLoop(cc).IsClosed()) {
+            kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          }
        }
        loop_packets_.pop_front();
        main_packet_specs_.pop_front();
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/timestamp.h"

 namespace mediapipe {
+namespace api2 {

 // A calculator that takes a packet of an input stream and converts it to an
 // output side packet. This calculator only works under the assumption that the
@@ -28,21 +30,21 @@ namespace mediapipe {
 //   input_stream: "stream"
 //   output_side_packet: "side_packet"
 // }
-class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+class StreamToSidePacketCalculator : public Node {
 public:
-  static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
-    cc->Inputs().Index(0).SetAny();
-    cc->OutputSidePackets().Index(0).SetAny();
-    return absl::OkStatus();
-  }
+  static constexpr Input<AnyType>::Optional kIn{""};
+  static constexpr SideOutput<SameType<kIn>> kOut{""};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);

  absl::Status Process(mediapipe::CalculatorContext* cc) override {
-    mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
-    cc->OutputSidePackets().Index(0).Set(
-        packet.At(mediapipe::Timestamp::Unset()));
+    kOut(cc).Set(
+        kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
    return absl::OkStatus();
  }
 };
-REGISTER_CALCULATOR(StreamToSidePacketCalculator);

+MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe
@@ -135,7 +135,6 @@ cc_library(
    deps = [
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/port:opencv_imgcodecs",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:status",
    ],
@@ -112,7 +112,7 @@ class BilateralFilterCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(BilateralFilterCalculator);

 absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);

  if (cc->Inputs().HasTag(kInputFrameTag) &&
      cc->Inputs().HasTag(kInputFrameTagGpu)) {
@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);

 absl::Status SegmentationSmoothingCalculator::GetContract(
    CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);

  cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
  cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(SetAlphaCalculator);

 absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);

  bool use_gpu = false;

@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
  buf[0] = (fourcc >> 24) & 0xff;
  buf[1] = (fourcc >> 16) & 0xff;
  buf[2] = (fourcc >> 8) & 0xff;
-  buf[3] = (fourcc)&0xff;
+  buf[3] = (fourcc) & 0xff;
  buf[4] = 0;
  return std::string(buf);
 }
@@ -228,7 +228,6 @@ cc_library(
        "//mediapipe/tasks/metadata:metadata_schema_cc",
        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
        "@com_google_absl//absl/strings",
    ],
    alwayslink = 1,
@@ -280,7 +279,6 @@ cc_library(
        "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
        "//mediapipe/tasks/metadata:metadata_schema_cc",
        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
    ],
    alwayslink = 1,
 )
@@ -282,18 +282,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
  if (options.has_volume_gain_db()) {
    gain_ = pow(10, options.volume_gain_db() / 20.0);
  }
-  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
-            !kAudioIn(cc).Header().IsEmpty())
-      << "Must either specify the time series header of the \"AUDIO\" stream "
-         "or have the \"SAMPLE_RATE\" stream connected.";
-  if (!kAudioIn(cc).Header().IsEmpty()) {
-    mediapipe::TimeSeriesHeader input_header;
-    MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
-        kAudioIn(cc).Header(), &input_header));
-    if (stream_mode_) {
-      MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
-    } else {
-      source_sample_rate_ = input_header.sample_rate();
+  if (options.has_source_sample_rate()) {
+    source_sample_rate_ = options.source_sample_rate();
+  } else {
+    RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+              !kAudioIn(cc).Header().IsEmpty())
+        << "Must either specify the time series header of the \"AUDIO\" stream "
+           "or have the \"SAMPLE_RATE\" stream connected.";
+    if (!kAudioIn(cc).Header().IsEmpty()) {
+      mediapipe::TimeSeriesHeader input_header;
+      MP_RETURN_IF_ERROR(
+          mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+              kAudioIn(cc).Header(), &input_header));
+      if (stream_mode_) {
+        MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      } else {
+        source_sample_rate_ = input_header.sample_rate();
+      }
    }
  }
  AppendZerosToSampleBuffer(padding_samples_before_);
@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
  // The volume gain, measured in dB.
  // Scale the input audio amplitude by 10^(volume_gain_db/20).
  optional double volume_gain_db = 12;
+
+  // The source number of samples per second (hertz) of the input audio buffers.
+  optional double source_sample_rate = 13;
 }
@@ -22,7 +22,6 @@

 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
@@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
  input_tensors.reserve(kNumInputTensorsForBert);
  for (int i = 0; i < kNumInputTensorsForBert; ++i) {
    input_tensors.push_back(
-        {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
+        {Tensor::ElementType::kInt32,
+         Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
  }
  std::memcpy(input_tensors[input_ids_tensor_index_]
                  .GetCpuWriteView()
@@ -69,6 +69,7 @@ class InferenceCalculatorGlAdvancedImpl
            gpu_delegate_options);
    absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
    absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
+    bool UseSerializedModel() const { return use_serialized_model_; }

   private:
    bool use_kernel_caching_ = false;
@@ -150,8 +151,6 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
 }

 absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
-  MP_RETURN_IF_ERROR(
-      on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
  return gpu_helper_.RunInGlContext([this]() -> absl::Status {
    tflite_gpu_runner_.reset();
    return absl::OkStatus();
@@ -226,9 +225,14 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
                         tflite_gpu_runner_->GetOutputShapes()[i].c};
  }

+  if (on_disk_cache_helper_.UseSerializedModel()) {
+    tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel();
+  }
+
  MP_RETURN_IF_ERROR(
      on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
-  return tflite_gpu_runner_->Build();
+  MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
+  return on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get());
 }

 #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
@@ -96,6 +96,19 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
    CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
  // Read CPU input into tensors.
  RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
+
+  // If the input tensors have dynamic shape, then the tensors need to be
+  // resized and reallocated before we can copy the tensor values.
+  bool resized_tensor_shapes = false;
+  for (int i = 0; i < input_tensors.size(); ++i) {
+    if (input_tensors[i].shape().is_dynamic) {
+      interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims);
+      resized_tensor_shapes = true;
+    }
+  }
+  // Reallocation is needed for memory sanity.
+  if (resized_tensor_shapes) interpreter_->AllocateTensors();
+
  for (int i = 0; i < input_tensors.size(); ++i) {
    const TfLiteType input_tensor_type =
        interpreter_->tensor(interpreter_->inputs()[i])->type;
@@ -20,7 +20,6 @@
 #include <vector>

 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
@@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
  // not found in the tokenizer vocab.
  std::vector<Tensor> result;
  result.push_back(
-      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
+      {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})});
  std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
              input_tokens.data(), input_tokens.size() * sizeof(int32_t));
  kTensorsOut(cc).Send(std::move(result));
@@ -256,6 +256,7 @@ class TensorsToDetectionsCalculator : public Node {

  bool gpu_inited_ = false;
  bool gpu_input_ = false;
+  bool gpu_has_enough_work_groups_ = true;
  bool anchors_init_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator);
@@ -291,7 +292,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
  auto output_detections = absl::make_unique<std::vector<Detection>>();
  bool gpu_processing = false;
-  if (CanUseGpu()) {
+  if (CanUseGpu() && gpu_has_enough_work_groups_) {
    // Use GPU processing only if at least one input tensor is already on GPU
    // (to avoid CPU->GPU overhead).
    for (const auto& tensor : *kInTensors(cc)) {
@@ -321,11 +322,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
    RET_CHECK(!has_custom_box_indices_);
  }

-  if (gpu_processing) {
-    if (!gpu_inited_) {
-      MP_RETURN_IF_ERROR(GpuInit(cc));
+  if (gpu_processing && !gpu_inited_) {
+    auto status = GpuInit(cc);
+    if (status.ok()) {
      gpu_inited_ = true;
+    } else if (status.code() == absl::StatusCode::kFailedPrecondition) {
+      // For initialization error because of hardware limitation, fallback to
+      // CPU processing.
+      LOG(WARNING) << status.message();
+    } else {
+      // For other error, let the error propagates.
+      return status;
    }
+  }
+  if (gpu_processing && gpu_inited_) {
    MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get()));
  } else {
    MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get()));
@@ -346,17 +356,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
    // TODO: Add flexible input tensor size handling.
    auto raw_box_tensor =
        &input_tensors[tensor_mapping_.detections_tensor_index()];
-    RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
    RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+    if (raw_box_tensor->shape().dims.size() == 3) {
+      // The tensors from CPU inference has dim 3.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+    } else if (raw_box_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of box Tensor must be 3 or 4.");
+    }
    auto raw_score_tensor =
        &input_tensors[tensor_mapping_.scores_tensor_index()];
-    RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+    if (raw_score_tensor->shape().dims.size() == 3) {
+      // The tensors from CPU inference has dim 3.
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+    } else if (raw_score_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of score Tensor must be 3 or 4.");
+    }
    auto raw_box_view = raw_box_tensor->GetCpuReadView();
    auto raw_boxes = raw_box_view.buffer<float>();
    auto raw_scores_view = raw_score_tensor->GetCpuReadView();
@@ -1111,8 +1145,13 @@ void main() {
    int max_wg_size;  //  typically <= 1024
    glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
                    &max_wg_size);  // y-dim
-    CHECK_LT(num_classes_, max_wg_size)
-        << "# classes must be < " << max_wg_size;
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
    // TODO support better filtering.
    if (class_index_set_.is_allowlist) {
      CHECK_EQ(class_index_set_.values.size(),
@@ -1370,7 +1409,13 @@ kernel void scoreKernel(
        Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2});
    // # filter classes supported is hardware dependent.
    int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup;
-    CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
  }

 #endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -406,8 +406,13 @@ cc_library(
    alwayslink = 1,
 )

-# This dependency removed tensorflow_jellyfish_deps and xprofilez_with_server because they failed
-# Boq conformance test. Weigh your use case to see if this will work for you.
+# This dependency removed the following 3 targets because they failed Boq conformance test:
+#
+# tensorflow_jellyfish_deps
+# jfprof_lib
+# xprofilez_with_server
+#
+# If you need them plz consider tensorflow_inference_calculator_no_envelope_loader.
 cc_library(
    name = "tensorflow_inference_calculator_for_boq",
    srcs = ["tensorflow_inference_calculator.cc"],
@@ -927,7 +932,6 @@ cc_test(
        "//mediapipe/framework:timestamp",
        "//mediapipe/framework/formats:detection_cc_proto",
        "//mediapipe/framework/formats:image_frame",
-        "//mediapipe/framework/formats:image_frame_opencv",
        "//mediapipe/framework/formats:location",
        "//mediapipe/framework/formats:location_opencv",
        "//mediapipe/framework/port:gtest_main",
@@ -1077,6 +1081,7 @@ cc_test(
    linkstatic = 1,
    deps = [
        ":tensor_to_image_frame_calculator",
+        ":tensor_to_image_frame_calculator_cc_proto",
        "//mediapipe/framework:calculator_framework",
        "//mediapipe/framework:calculator_runner",
        "//mediapipe/framework/formats:image_frame",
@@ -164,8 +164,8 @@ class PackMediaSequenceCalculator : public CalculatorBase {
      }
    }

-    CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
-          cc->OutputSidePackets().HasTag(kSequenceExampleTag))
+    RET_CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
+              cc->OutputSidePackets().HasTag(kSequenceExampleTag))
        << "Neither the output stream nor the output side packet is set to "
           "output the sequence example.";
    if (cc->Outputs().HasTag(kSequenceExampleTag)) {
@@ -23,7 +23,6 @@
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/location.h"
 #include "mediapipe/framework/formats/location_opencv.h"
 #include "mediapipe/framework/port/gmock.h"
@@ -96,7 +95,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
  mpms::SetClipMediaId(test_video_id, input_sequence.get());
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  encoded_image.set_width(2);
@@ -139,7 +139,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
  mpms::SetClipMediaId(test_video_id, input_sequence.get());
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  encoded_image.set_width(2);
@@ -378,7 +379,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
      Adopt(input_sequence.release());
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  auto image_ptr =
@@ -410,7 +412,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {

  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  std::string test_flow_string(bytes.begin(), bytes.end());
  OpenCvImageEncoderCalculatorResults encoded_flow;
  encoded_flow.set_encoded_image(test_flow_string);
@@ -618,7 +621,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
  }
  cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  encoded_image.set_width(width);
@@ -767,7 +771,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {

  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  std::string test_flow_string(bytes.begin(), bytes.end());
  OpenCvImageEncoderCalculatorResults encoded_flow;
  encoded_flow.set_encoded_image(test_flow_string);
@@ -813,7 +818,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
  mpms::SetClipMediaId(test_video_id, input_sequence.get());
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  std::string test_flow_string(bytes.begin(), bytes.end());
  OpenCvImageEncoderCalculatorResults encoded_flow;
  encoded_flow.set_encoded_image(test_flow_string);
@@ -970,7 +976,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  encoded_image.set_width(2);
@@ -1021,7 +1028,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
  auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
  cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
  std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
  OpenCvImageEncoderCalculatorResults encoded_image;
  encoded_image.set_encoded_image(bytes.data(), bytes.size());
  int height = 2;
@@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase {

 private:
  float scale_factor_;
+  bool scale_per_frame_min_max_;
 };

 REGISTER_CALCULATOR(TensorToImageFrameCalculator);
@@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
 absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) {
  scale_factor_ =
      cc->Options<TensorToImageFrameCalculatorOptions>().scale_factor();
+  scale_per_frame_min_max_ = cc->Options<TensorToImageFrameCalculatorOptions>()
+                                 .scale_per_frame_min_max();
  cc->SetOffset(TimestampDiff(0));
  return absl::OkStatus();
 }
@@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
  auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
  const int32_t total_size = height * width * depth;

+  if (scale_per_frame_min_max_) {
+    RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT)
+        << "Setting scale_per_frame_min_max requires FLOAT input tensors.";
+  }
  ::std::unique_ptr<const ImageFrame> output;
  if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
    // Allocate buffer with alignments.
    std::unique_ptr<uint8_t[]> buffer(
        new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
    auto data = input_tensor.flat<float>().data();
+    float min = 1e23;
+    float max = -1e23;
+    if (scale_per_frame_min_max_) {
+      for (int i = 0; i < total_size; ++i) {
+        float d = scale_factor_ * data[i];
+        if (d < min) {
+          min = d;
+        }
+        if (d > max) {
+          max = d;
+        }
+      }
+    }
    for (int i = 0; i < total_size; ++i) {
-      float d = scale_factor_ * data[i];
-      if (d < 0) d = 0;
-      if (d > 255) d = 255;
+      float d = data[i];
+      if (scale_per_frame_min_max_) {
+        d = 255 * (d - min) / (max - min + 1e-9);
+      } else {
+        d = scale_factor_ * d;
+        if (d < 0) d = 0;
+        if (d > 255) d = 255;
+      }
      buffer[i] = d;
    }
    output = ::absl::make_unique<ImageFrame>(
@@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions {
  // Multiples floating point tensor outputs by this value before converting to
  // uint8. This is useful for converting from range [0, 1] to [0, 255]
  optional float scale_factor = 1 [default = 1.0];
+
+  // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255]
+  // per frame. This overrides any explicit scale_factor.
+  optional bool scale_per_frame_min_max = 2 [default = false];
 }
@@ -11,7 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <type_traits>

+#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE";
 template <class TypeParam>
 class TensorToImageFrameCalculatorTest : public ::testing::Test {
 protected:
-  void SetUpRunner() {
+  void SetUpRunner(bool scale_per_frame_min_max = false) {
    CalculatorGraphConfig::Node config;
    config.set_calculator("TensorToImageFrameCalculator");
    config.add_input_stream("TENSOR:input_tensor");
    config.add_output_stream("IMAGE:output_image");
+    config.mutable_options()
+        ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext)
+        ->set_scale_per_frame_min_max(scale_per_frame_min_max);
    runner_ = absl::make_unique<CalculatorRunner>(config);
  }

@@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest,
  }
 }

+TYPED_TEST(TensorToImageFrameCalculatorTest,
+           Converts3DTensorToImageFrame2DGrayWithScaling) {
+  this->SetUpRunner(true);
+  auto& runner = this->runner_;
+  constexpr int kWidth = 16;
+  constexpr int kHeight = 8;
+  const tf::TensorShape tensor_shape{kHeight, kWidth};
+  auto tensor = absl::make_unique<tf::Tensor>(
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
+
+  // Writing sequence of integers as floats which we want normalized.
+  tensor_vec[0] = 255;
+  for (int i = 1; i < kWidth * kHeight; ++i) {
+    tensor_vec[i] = 200;
+  }
+
+  const int64_t time = 1234;
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  if (!std::is_same<TypeParam, float>::value) {
+    EXPECT_FALSE(runner->Run().ok());
+    return;  // Short circuit because does not apply to other types.
+  } else {
+    EXPECT_TRUE(runner->Run().ok());
+    const std::vector<Packet>& output_packets =
+        runner->Outputs().Tag(kImage).packets;
+    EXPECT_EQ(1, output_packets.size());
+    EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+    const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+    EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
+    EXPECT_EQ(kWidth, output_image.Width());
+    EXPECT_EQ(kHeight, output_image.Height());
+
+    EXPECT_EQ(255, output_image.PixelData()[0]);
+    for (int i = 1; i < kWidth * kHeight; ++i) {
+      const uint8_t pixel_value = output_image.PixelData()[i];
+      ASSERT_EQ(0, pixel_value);
+    }
+  }
+}
+
 }  // namespace mediapipe
@@ -172,7 +172,7 @@ class AnnotationOverlayCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(AnnotationOverlayCalculator);

 absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);

  bool use_gpu = false;

@@ -189,13 +189,13 @@ absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
 #if !MEDIAPIPE_DISABLE_GPU
  if (cc->Inputs().HasTag(kGpuBufferTag)) {
    cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
-    CHECK(cc->Outputs().HasTag(kGpuBufferTag));
+    RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
    use_gpu = true;
  }
 #endif  // !MEDIAPIPE_DISABLE_GPU
  if (cc->Inputs().HasTag(kImageFrameTag)) {
    cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    CHECK(cc->Outputs().HasTag(kImageFrameTag));
+    RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
  }

  // Data streams to render.
@@ -322,27 +322,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
          options_.presence_threshold(), options_.connection_color(), thickness,
          /*normalized=*/false, render_data.get());
    }
-    for (int i = 0; i < landmarks.landmark_size(); ++i) {
-      const Landmark& landmark = landmarks.landmark(i);
+    if (options_.render_landmarks()) {
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        const Landmark& landmark = landmarks.landmark(i);

-      if (!IsLandmarkVisibleAndPresent<Landmark>(
-              landmark, options_.utilize_visibility(),
-              options_.visibility_threshold(), options_.utilize_presence(),
-              options_.presence_threshold())) {
-        continue;
-      }
+        if (!IsLandmarkVisibleAndPresent<Landmark>(
+                landmark, options_.utilize_visibility(),
+                options_.visibility_threshold(), options_.utilize_presence(),
+                options_.presence_threshold())) {
+          continue;
+        }

-      auto* landmark_data_render = AddPointRenderData(
-          options_.landmark_color(), thickness, render_data.get());
-      if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
-                               options_.min_depth_circle_thickness(),
-                               options_.max_depth_circle_thickness());
+        auto* landmark_data_render = AddPointRenderData(
+            options_.landmark_color(), thickness, render_data.get());
+        if (visualize_depth) {
+          SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
+                                 landmark_data_render,
+                                 options_.min_depth_circle_thickness(),
+                                 options_.max_depth_circle_thickness());
+        }
+        auto* landmark_data = landmark_data_render->mutable_point();
+        landmark_data->set_normalized(false);
+        landmark_data->set_x(landmark.x());
+        landmark_data->set_y(landmark.y());
      }
-      auto* landmark_data = landmark_data_render->mutable_point();
-      landmark_data->set_normalized(false);
-      landmark_data->set_x(landmark.x());
-      landmark_data->set_y(landmark.y());
    }
  }

@@ -368,27 +371,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
          options_.presence_threshold(), options_.connection_color(), thickness,
          /*normalized=*/true, render_data.get());
    }
-    for (int i = 0; i < landmarks.landmark_size(); ++i) {
-      const NormalizedLandmark& landmark = landmarks.landmark(i);
+    if (options_.render_landmarks()) {
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        const NormalizedLandmark& landmark = landmarks.landmark(i);

-      if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
-              landmark, options_.utilize_visibility(),
-              options_.visibility_threshold(), options_.utilize_presence(),
-              options_.presence_threshold())) {
-        continue;
-      }
+        if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
+                landmark, options_.utilize_visibility(),
+                options_.visibility_threshold(), options_.utilize_presence(),
+                options_.presence_threshold())) {
+          continue;
+        }

-      auto* landmark_data_render = AddPointRenderData(
-          options_.landmark_color(), thickness, render_data.get());
-      if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
-                               options_.min_depth_circle_thickness(),
-                               options_.max_depth_circle_thickness());
+        auto* landmark_data_render = AddPointRenderData(
+            options_.landmark_color(), thickness, render_data.get());
+        if (visualize_depth) {
+          SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
+                                 landmark_data_render,
+                                 options_.min_depth_circle_thickness(),
+                                 options_.max_depth_circle_thickness());
+        }
+        auto* landmark_data = landmark_data_render->mutable_point();
+        landmark_data->set_normalized(true);
+        landmark_data->set_x(landmark.x());
+        landmark_data->set_y(landmark.y());
      }
-      auto* landmark_data = landmark_data_render->mutable_point();
-      landmark_data->set_normalized(true);
-      landmark_data->set_x(landmark.x());
-      landmark_data->set_y(landmark.y());
    }
  }

@@ -32,6 +32,10 @@ message LandmarksToRenderDataCalculatorOptions {

  // Color of the landmarks.
  optional Color landmark_color = 2;
+
+  // Whether to render landmarks as points.
+  optional bool render_landmarks = 14 [default = true];
+
  // Color of the connections.
  optional Color connection_color = 3;

@@ -124,7 +124,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
    int center_row = out_lms.landmark(lm_index).y() * hm_height;
    // Point is outside of the image let's keep it intact.
    if (center_col < 0 || center_col >= hm_width || center_row < 0 ||
-        center_col >= hm_height) {
+        center_row >= hm_height) {
      continue;
    }

@@ -130,7 +130,6 @@ cc_library(
        "//mediapipe/framework/formats:video_stream_header",
        "//mediapipe/framework/port:opencv_imgproc",
        "//mediapipe/framework/port:opencv_video",
-        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
        "//mediapipe/framework/tool:status_util",
    ],
@@ -341,7 +340,6 @@ cc_test(
        "//mediapipe/framework/port:opencv_core",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/tool:test_util",
-        "@com_google_absl//absl/flags:flag",
    ],
 )

@@ -367,7 +365,6 @@ cc_test(
        "//mediapipe/framework/port:opencv_video",
        "//mediapipe/framework/port:parse_text_proto",
        "//mediapipe/framework/tool:test_util",
-        "@com_google_absl//absl/flags:flag",
    ],
 )

@@ -451,7 +448,6 @@ cc_test(
        "//mediapipe/framework/tool:test_util",
        "//mediapipe/util/tracking:box_tracker_cc_proto",
        "//mediapipe/util/tracking:tracking_cc_proto",
-        "@com_google_absl//absl/flags:flag",
    ],
 )

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "facedetectioncpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "facedetectiongpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "faceeffect",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "facemeshgpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "handdetectiongpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "handtrackinggpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "helloworld",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "holistictrackinggpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "iristrackinggpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "objectdetectioncpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "objectdetectiongpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "objectdetectiontrackinggpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "posetrackinggpu",
@@ -24,7 +24,7 @@ load(

 licenses(["notice"])

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 alias(
    name = "selfiesegmentationgpu",
@@ -44,6 +44,9 @@ bzl_library(
        "encode_binary_proto.bzl",
    ],
    visibility = ["//visibility:public"],
+    deps = [
+        "@bazel_skylib//lib:paths",
+    ],
 )

 alias(
@@ -1355,6 +1358,23 @@ cc_test(
    ],
 )

+cc_test(
+    name = "calculator_graph_summary_packet_test",
+    srcs = ["calculator_graph_summary_packet_test.cc"],
+    deps = [
+        ":calculator_framework",
+        ":packet",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_test(
    name = "calculator_runner_test",
    size = "medium",
@@ -32,7 +32,7 @@ template <class T>
 struct dependent_false : std::false_type {};

 template <typename T>
-T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, int index) {
+T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, size_t index) {
  auto& vec = *vecp;
  if (vec.size() <= index) {
    vec.resize(index + 1);
@@ -64,58 +64,13 @@ class CalculatorBaseFactoryFor<
 namespace api2 {
 namespace internal {

-// Defining a member of this type causes P to be ODR-used, which forces its
-// instantiation if it's a static member of a template.
-// Previously we depended on the pointer's value to determine whether the size
-// of a character array is 0 or 1, forcing it to be instantiated so the
-// compiler can determine the object's layout. But using it as a template
-// argument is more compact.
-template <auto* P>
-struct ForceStaticInstantiation {
-#ifdef _MSC_VER
-  // Just having it as the template argument does not count as a use for
-  // MSVC.
-  static constexpr bool Use() { return P != nullptr; }
-  char force_static[Use()];
-#endif  // _MSC_VER
-};
+MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(
+    NodeRegistrator, mediapipe::CalculatorBaseRegistry, T::kCalculatorName,
+    absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>)

-// Helper template for forcing the definition of a static registration token.
-template <typename T>
-struct NodeRegistrationStatic {
-  static NoDestructor<mediapipe::RegistrationToken> registration;
-
-  static mediapipe::RegistrationToken Make() {
-    return mediapipe::CalculatorBaseRegistry::Register(
-        T::kCalculatorName,
-        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>,
-        __FILE__, __LINE__);
-  }
-
-  using RequireStatics = ForceStaticInstantiation<&registration>;
-};
-
-// Static members of template classes can be defined in the header.
-template <typename T>
-NoDestructor<mediapipe::RegistrationToken>
-    NodeRegistrationStatic<T>::registration(NodeRegistrationStatic<T>::Make());
-
-template <typename T>
-struct SubgraphRegistrationImpl {
-  static NoDestructor<mediapipe::RegistrationToken> registration;
-
-  static mediapipe::RegistrationToken Make() {
-    return mediapipe::SubgraphRegistry::Register(
-        T::kCalculatorName, absl::make_unique<T>, __FILE__, __LINE__);
-  }
-
-  using RequireStatics = ForceStaticInstantiation<&registration>;
-};
-
-template <typename T>
-NoDestructor<mediapipe::RegistrationToken>
-    SubgraphRegistrationImpl<T>::registration(
-        SubgraphRegistrationImpl<T>::Make());
+MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator,
+                                      mediapipe::SubgraphRegistry,
+                                      T::kCalculatorName, absl::make_unique<T>)

 }  // namespace internal

@@ -128,14 +83,7 @@ template <class Impl = void>
 class RegisteredNode;

 template <class Impl>
-class RegisteredNode : public Node {
- private:
-  // The member below triggers instantiation of the registration static.
-  // Note that the constructor of calculator subclasses is only invoked through
-  // the registration token, and so we cannot simply use the static in the
-  // constructor.
-  typename internal::NodeRegistrationStatic<Impl>::RequireStatics register_;
-};
+class RegisteredNode : public Node, private internal::NodeRegistrator<Impl> {};

 // No-op version for backwards compatibility.
 template <>
@@ -217,31 +165,27 @@ class NodeImpl : public RegisteredNode<Impl>, public Intf {
 // TODO: verify that the subgraph config fully implements the
 // declared interface.
 template <class Intf, class Impl>
-class SubgraphImpl : public Subgraph, public Intf {
- private:
-  typename internal::SubgraphRegistrationImpl<Impl>::RequireStatics register_;
-};
+class SubgraphImpl : public Subgraph,
+                     public Intf,
+                     private internal::SubgraphRegistrator<Impl> {};

 // This macro is used to register a calculator that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                   \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>                \
-  REGISTRY_STATIC_VAR(calculator_registration,                                \
-                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register(  \
-      Impl::kCalculatorName,                                                  \
-      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>, \
-      __FILE__, __LINE__))
+#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                       \
+  MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(                  \
+      mediapipe::CalculatorBaseRegistry, calculator_registration, \
+      Impl::kCalculatorName,                                      \
+      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>)

 // This macro is used to register a non-split-contract calculator. Deprecated.
 #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)

 // This macro is used to define a subgraph that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)                        \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>         \
-  REGISTRY_STATIC_VAR(subgraph_registration,                           \
-                      __LINE__)(mediapipe::SubgraphRegistry::Register( \
-      Impl::kCalculatorName, absl::make_unique<Impl>, __FILE__, __LINE__))
+#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)           \
+  MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(          \
+      mediapipe::SubgraphRegistry, subgraph_registration, \
+      Impl::kCalculatorName, absl::make_unique<Impl>)

 }  // namespace api2
 }  // namespace mediapipe
@@ -183,8 +183,7 @@ TEST(CalculatorTest, CreateByNameWhitelisted) {
  CalculatorBaseRegistry::Register(
      "::mediapipe::test_ns::whitelisted_ns::DeadCalculator",
      absl::make_unique<internal::CalculatorBaseFactoryFor<
-          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>,
-      __FILE__, __LINE__);
+          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>);

  // A whitelisted calculator can be found in its own namespace.
  MP_EXPECT_OK(CalculatorBaseRegistry::CreateByNameInNamespace(  //
@@ -109,9 +109,20 @@ class CalculatorContext {
  // use OutputStream::SetOffset() directly.
  void SetOffset(TimestampDiff offset);

-  // Returns the status of the graph run.
+  // DEPRECATED: This was intended to get graph run status during
+  // `CalculatorBase::Close` call. However, `Close` can run simultaneously with
+  // other calculators `CalculatorBase::Process`, hence the actual graph
+  // status may change any time and returned graph status here does not
+  // necessarily reflect the actual graph status.
  //
-  // NOTE: This method should only be called during CalculatorBase::Close().
+  // As an alternative, instead of checking graph status in `Close` and doing
+  // work for "done" state, you can enable timestamp bound processing for your
+  // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger
+  // `Process` on timestamp bound updates and handle "done" state there.
+  // Check examples in:
+  // mediapipe/framework/calculator_graph_summary_packet_test.cc.
+  //
+  ABSL_DEPRECATED("Does not reflect the actual graph status.")
  absl::Status GraphStatus() const { return graph_status_; }

  ProfilingContext* GetProfilingContext() const {
@@ -839,6 +839,13 @@ absl::Status CalculatorGraph::PrepareForRun(
 }

 absl::Status CalculatorGraph::WaitUntilIdle() {
+  if (has_sources_) {
+    LOG_FIRST_N(WARNING, 1)
+        << "WaitUntilIdle called on a graph with source nodes, which "
+           "is not fully supported at the moment. Source nodes: "
+        << ListSourceNodes();
+  }
+
  MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle());
  VLOG(2) << "Scheduler idle.";
  absl::Status status = absl::OkStatus();
@@ -1368,6 +1375,16 @@ const OutputStreamManager* CalculatorGraph::FindOutputStreamManager(
              .get()[validated_graph_->OutputStreamIndex(name)];
 }

+std::string CalculatorGraph::ListSourceNodes() const {
+  std::vector<std::string> sources;
+  for (auto& node : nodes_) {
+    if (node->IsSource()) {
+      sources.push_back(node->DebugName());
+    }
+  }
+  return absl::StrJoin(sources, ", ");
+}
+
 namespace {
 void PrintTimingToInfo(const std::string& label, int64_t timer_value) {
  const int64_t total_seconds = timer_value / 1000000ll;
@@ -229,8 +229,11 @@ class CalculatorGraph {
  // Wait until the running graph is in the idle mode, which is when nothing can
  // be scheduled and nothing is running in the worker threads. This function
  // can be called only after StartRun().
+  //
  // NOTE: The graph must not have any source nodes because source nodes prevent
  // the running graph from becoming idle until the source nodes are done.
+  // Currently, `WaitUntilIdle` cannot be used reliably on graphs with any
+  // source nodes.
  absl::Status WaitUntilIdle();

  // Wait until a packet is emitted on one of the observed output streams.
@@ -594,6 +597,9 @@ class CalculatorGraph {
  // status before taking any action.
  void UpdateThrottledNodes(InputStreamManager* stream, bool* stream_was_full);

+  // Returns a comma-separated list of source nodes.
+  std::string ListSourceNodes() const;
+
 #if !MEDIAPIPE_DISABLE_GPU
  // Owns the legacy GpuSharedData if we need to create one for backwards
  // compatibility.
@@ -0,0 +1,430 @@
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+using ::mediapipe::api2::Input;
+using ::mediapipe::api2::Node;
+using ::mediapipe::api2::Output;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsEmpty;
+using ::testing::Value;
+
+namespace {
+
+MATCHER_P2(IntPacket, value, timestamp, "") {
+  *result_listener << "where object is (value: " << arg.template Get<int>()
+                   << ", timestamp: " << arg.Timestamp() << ")";
+  return Value(arg.template Get<int>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(timestamp));
+}
+
+// Calculates and produces sum of all passed inputs when no more packets can be
+// expected on the input stream.
+class SummaryPacketCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"SUMMARY"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    // Makes sure there are no automatic timestamp bound updates when Process
+    // is called.
+    cc->SetTimestampOffset(TimestampDiff::Unset());
+    // Currently, only ImmediateInputStreamHandler supports "done" timestamp
+    // bound update. (ImmediateInputStreamhandler handles multiple input
+    // streams differently, so, in that case, calculator adjustments may be
+    // required.)
+    // TODO: update all input stream handlers to support "done"
+    // timestamp bound update.
+    cc->SetInputStreamHandler("ImmediateInputStreamHandler");
+    // Enables processing timestamp bound updates. For this use case we are
+    // specifically interested in "done" timestamp bound update. (E.g. when
+    // all input packet sources are closed.)
+    cc->SetProcessTimestampBounds(true);
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      value_ += kIn(cc).Get();
+      value_set_ = true;
+    }
+
+    if (kOut(cc).IsClosed()) {
+      // This can happen:
+      // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g.
+      //    source calculator finished generating packets sent to kIn) and
+      //    HasNextAllowedInStream() == true (which is an often case).
+      // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still
+      //    invoke Process() with Timestamp::Max to indicate "Done" timestamp
+      //    bound update.
+      return absl::OkStatus();
+    }
+
+    // TODO: input stream holding a packet with timestamp that has
+    // no next timestamp allowed in stream should always result in
+    // InputStream::IsDone() == true.
+    if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) {
+      // `Process` may or may not be invoked for "done" timestamp bound when
+      // upstream calculator fails in `Close`. Hence, extra care is needed to
+      // identify whether the calculator needs to send output.
+      // TODO: remove when "done" timestamp bound flakiness fixed.
+      if (value_set_) {
+        // kOut(cc).Send(value_) can be used here as well, however in the case
+        // of source calculator sending inputs into kIn the resulting timestamp
+        // is not well defined (e.g. it can be the last packet timestamp or
+        // Timestamp::Max())
+        // TODO: last packet from source should always result in
+        // InputStream::IsDone() == true.
+        kOut(cc).Send(value_, Timestamp::Max());
+      }
+      kOut(cc).Close();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  int value_ = 0;
+  bool value_set_ = false;
+};
+MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnClosingAllPacketSources) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp(11));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp::Max());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPreStreamTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PreStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPostStreamTimestamp) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input'
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: 'IN:input'
+          output_stream: 'SUMMARY:output'
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class IntGeneratorCalculator : public Node {
+ public:
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return tool::StatusStop();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnSourceCalculatorCompletion) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        node {
+          calculator: "IntGeneratorCalculator"
+          output_stream: "INT:int_value"
+        }
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: "IN:int_value"
+          output_stream: "SUMMARY:output"
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+class EmitOnCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnAnotherCalculatorClosure) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "EmitOnCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInClose) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInProcessCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInProcessCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInProcess) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInProcessCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+}  // namespace
+}  // namespace mediapipe
@@ -16,7 +16,6 @@
 #define MEDIAPIPE_DEPS_REGISTRATION_H_

 #include <algorithm>
-#include <cstdint>
 #include <functional>
 #include <string>
 #include <tuple>
@@ -145,6 +144,23 @@ template <typename T>
 struct WrapStatusOr<absl::StatusOr<T>> {
  using type = absl::StatusOr<T>;
 };
+
+// Defining a member of this type causes P to be ODR-used, which forces its
+// instantiation if it's a static member of a template.
+// Previously we depended on the pointer's value to determine whether the size
+// of a character array is 0 or 1, forcing it to be instantiated so the
+// compiler can determine the object's layout. But using it as a template
+// argument is more compact.
+template <auto* P>
+struct ForceStaticInstantiation {
+#ifdef _MSC_VER
+  // Just having it as the template argument does not count as a use for
+  // MSVC.
+  static constexpr bool Use() { return P != nullptr; }
+  char force_static[Use()];
+#endif  // _MSC_VER
+};
+
 }  // namespace registration_internal

 class NamespaceAllowlist {
@@ -162,8 +178,7 @@ class FunctionRegistry {
  FunctionRegistry(const FunctionRegistry&) = delete;
  FunctionRegistry& operator=(const FunctionRegistry&) = delete;

-  RegistrationToken Register(absl::string_view name, Function func,
-                             std::string filename, uint64_t line)
+  RegistrationToken Register(absl::string_view name, Function func)
      ABSL_LOCKS_EXCLUDED(lock_) {
    std::string normalized_name = GetNormalizedName(name);
    absl::WriterMutexLock lock(&lock_);
@@ -173,21 +188,10 @@ class FunctionRegistry {
    }
    if (functions_.insert(std::make_pair(normalized_name, std::move(func)))
            .second) {
-#ifndef NDEBUG
-      locations_.emplace(normalized_name,
-                         std::make_pair(std::move(filename), line));
-#endif
      return RegistrationToken(
          [this, normalized_name]() { Unregister(normalized_name); });
    }
-#ifndef NDEBUG
-    LOG(FATAL) << "Function with name " << name << " already registered."
-               << " First registration at "
-               << locations_.at(normalized_name).first << ":"
-               << locations_.at(normalized_name).second;
-#else
    LOG(FATAL) << "Function with name " << name << " already registered.";
-#endif
    return RegistrationToken([]() {});
  }

@@ -316,11 +320,6 @@ class FunctionRegistry {
 private:
  mutable absl::Mutex lock_;
  absl::flat_hash_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
-#ifndef NDEBUG
-  // Stores filename and line number for useful debug log.
-  absl::flat_hash_map<std::string, std::pair<std::string, uint32_t>> locations_
-      ABSL_GUARDED_BY(lock_);
-#endif

  // For names included in NamespaceAllowlist, strips the namespace.
  std::string GetAdjustedName(absl::string_view name) {
@@ -351,10 +350,8 @@ class GlobalFactoryRegistry {

 public:
  static RegistrationToken Register(absl::string_view name,
-                                    typename Functions::Function func,
-                                    std::string filename, uint64_t line) {
-    return functions()->Register(name, std::move(func), std::move(filename),
-                                 line);
+                                    typename Functions::Function func) {
+    return functions()->Register(name, std::move(func));
  }

  // Invokes the specified factory function and returns the result.
@@ -414,12 +411,77 @@ class GlobalFactoryRegistry {
 #define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \
  static auto* REGISTRY_STATIC_VAR(registration_##name, __LINE__) =  \
      new mediapipe::RegistrationToken(                              \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))

+#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, \
+                                                      name, ...)              \
+  static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                      \
+      new mediapipe::RegistrationToken(                                       \
+          RegistryType::Register(name, __VA_ARGS__))
+
+// TODO: migrate to the above.
 #define REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, name, ...) \
  static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                       \
      new mediapipe::RegistrationToken(                                        \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
+
+// Defines a utility registrator class which can be used to automatically
+// register factory functions.
+//
+// Example:
+// === Defining a registry ================================================
+//
+//  class Component {};
+//
+//  using ComponentRegistry = GlobalFactoryRegistry<std::unique_ptr<Component>>;
+//
+// === Defining a registrator =============================================
+//
+//  MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(ComponentRegistrator,
+//                                        ComponentRegistry, T::kName,
+//                                        absl::make_unique<T>);
+//
+// === Defining and registering a new component. ==========================
+//
+//  class MyComponent : public Component,
+//                      private ComponentRegistrator<MyComponent> {
+//   public:
+//    static constexpr char kName[] = "MyComponent";
+//    ...
+//  };
+//
+// NOTE:
+// - MyComponent is automatically registered in ComponentRegistry by
+//   "MyComponent" name.
+// - Every component is require to provide its name (T::kName here.)
+#define MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(RegistratorName, RegistryType,  \
+                                              name, ...)                      \
+  template <typename T>                                                       \
+  struct Internal##RegistratorName {                                          \
+    static NoDestructor<mediapipe::RegistrationToken> registration;           \
+                                                                              \
+    static mediapipe::RegistrationToken Make() {                              \
+      return RegistryType::Register(name, __VA_ARGS__);                       \
+    }                                                                         \
+                                                                              \
+    using RequireStatics =                                                    \
+        registration_internal::ForceStaticInstantiation<&registration>;       \
+  };                                                                          \
+  /* Static members of template classes can be defined in the header. */      \
+  template <typename T>                                                       \
+  NoDestructor<mediapipe::RegistrationToken>                                  \
+      Internal##RegistratorName<T>::registration(                             \
+          Internal##RegistratorName<T>::Make());                              \
+                                                                              \
+  template <typename T>                                                       \
+  class RegistratorName {                                                     \
+   private:                                                                   \
+    /* The member below triggers instantiation of the registration static. */ \
+    /* Note that the constructor of calculator subclasses is only invoked  */ \
+    /* through the registration token, and so we cannot simply use the     */ \
+    /* static in theconstructor.                                           */ \
+    typename Internal##RegistratorName<T>::RequireStatics register_;          \
+  };

 }  // namespace mediapipe

@@ -37,29 +37,33 @@ Args:
  output: The desired name of the output file. Optional.
 """

+load("@bazel_skylib//lib:paths.bzl", "paths")
+
 PROTOC = "@com_google_protobuf//:protoc"

-def _canonicalize_proto_path_oss(all_protos, genfile_path):
-    """For the protos from external repository, canonicalize the proto path and the file name.
+def _canonicalize_proto_path_oss(f):
+    if not f.root.path:
+        return struct(
+            proto_path = ".",
+            file_name = f.short_path,
+        )

-    Returns:
-       Proto path list and proto source file list.
-    """
-    proto_paths = []
-    proto_file_names = []
-    for s in all_protos.to_list():
-        if s.path.startswith(genfile_path):
-            repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/")
+    # `f.path` looks like "<genfiles>/external/<repo>/(_virtual_imports/<library>/)?<file_name>"
+    repo_name, _, file_name = f.path[len(paths.join(f.root.path, "external") + "/"):].partition("/")
+    if file_name.startswith("_virtual_imports/"):
+        # This is a virtual import; move "_virtual_imports/<library>" from `repo_name` to `file_name`.
+        repo_name = paths.join(repo_name, *file_name.split("/", 2)[:2])
+        file_name = file_name.split("/", 2)[-1]
+    return struct(
+        proto_path = paths.join(f.root.path, "external", repo_name),
+        file_name = file_name,
+    )

-            # handle virtual imports
-            if file_name.startswith("_virtual_imports"):
-                repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2])
-                file_name = file_name.split("/", 2)[-1]
-            proto_paths.append(genfile_path + "/external/" + repo_name)
-            proto_file_names.append(file_name)
-        else:
-            proto_file_names.append(s.path)
-    return ([" --proto_path=" + path for path in proto_paths], proto_file_names)
+def _map_root_path(f):
+    return _canonicalize_proto_path_oss(f).proto_path
+
+def _map_short_path(f):
+    return _canonicalize_proto_path_oss(f).file_name

 def _get_proto_provider(dep):
    """Get the provider for protocol buffers from a dependnecy.
@@ -90,24 +94,35 @@ def _encode_binary_proto_impl(ctx):
        sibling = textpb,
    )

-    path_list, file_list = _canonicalize_proto_path_oss(all_protos, ctx.genfiles_dir.path)
+    args = ctx.actions.args()
+    args.add(textpb)
+    args.add(binarypb)
+    args.add(ctx.executable._proto_compiler)
+    args.add(ctx.attr.message_type, format = "--encode=%s")
+    args.add("--proto_path=.")
+    args.add_all(
+        all_protos,
+        map_each = _map_root_path,
+        format_each = "--proto_path=%s",
+        uniquify = True,
+    )
+    args.add_all(
+        all_protos,
+        map_each = _map_short_path,
+        uniquify = True,
+    )

    # Note: the combination of absolute_paths and proto_path, as well as the exact
    # order of gendir before ., is needed for the proto compiler to resolve
    # import statements that reference proto files produced by a genrule.
    ctx.actions.run_shell(
-        tools = all_protos.to_list() + [textpb, ctx.executable._proto_compiler],
-        outputs = [binarypb],
-        command = " ".join(
-            [
-                ctx.executable._proto_compiler.path,
-                "--encode=" + ctx.attr.message_type,
-                "--proto_path=" + ctx.genfiles_dir.path,
-                "--proto_path=" + ctx.bin_dir.path,
-                "--proto_path=.",
-            ] + path_list + file_list +
-            ["<", textpb.path, ">", binarypb.path],
+        tools = depset(
+            direct = [textpb, ctx.executable._proto_compiler],
+            transitive = [all_protos],
        ),
+        outputs = [binarypb],
+        command = "${@:3} < $1 > $2",
+        arguments = [args],
        mnemonic = "EncodeProto",
    )

@@ -19,7 +19,7 @@ package mediapipe;
 // Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of
 // the joint and its visibility.
 message Joint {
-  // Joint rotation in 6D contineous representation ordered as
+  // Joint rotation in 6D continuous representation ordered as
  // [a1, b1, a2, b2, a3, b3].
  //
  // Such representation is more sutable for NN model training and can be
@@ -117,11 +117,18 @@ class Tensor {
    Shape() = default;
    Shape(std::initializer_list<int> dimensions) : dims(dimensions) {}
    Shape(const std::vector<int>& dimensions) : dims(dimensions) {}
+    Shape(std::initializer_list<int> dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
+    Shape(const std::vector<int>& dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
    int num_elements() const {
      return std::accumulate(dims.begin(), dims.end(), 1,
                             std::multiplies<int>());
    }
    std::vector<int> dims;
+    // The Tensor has dynamic rather than static shape so the TFLite interpreter
+    // needs to be reallocated. Only relevant for CPU.
+    bool is_dynamic = false;
  };
  // Quantization parameters corresponding to the zero_point and scale value
  // made available by TfLite quantized (uint8/int8) tensors.
@@ -2,6 +2,7 @@

 #include <cstring>
 #include <string>
+#include <vector>

 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -34,6 +35,17 @@ TEST(General, TestDataTypes) {
  EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool));
 }

+TEST(General, TestDynamic) {
+  Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true));
+  EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4);
+  EXPECT_TRUE(t1.shape().is_dynamic);
+
+  std::vector<int> t2_dims = {4, 3, 2, 3};
+  Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true));
+  EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3);
+  EXPECT_TRUE(t2.shape().is_dynamic);
+}
+
 TEST(Cpu, TestMemoryAllocation) {
  Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3});
  auto v1 = t1.GetCpuWriteView();
@@ -15,7 +15,7 @@ def mediapipe_cc_test(
        platforms = ["linux", "android", "ios", "wasm"],
        exclude_platforms = None,
        # ios_unit_test arguments
-        ios_minimum_os_version = "11.0",
+        ios_minimum_os_version = "12.0",
        # android_cc_test arguments
        open_gl_driver = None,
        emulator_mini_boot = True,
@@ -466,8 +466,7 @@ struct MessageRegistrationImpl {
 template <typename T>
 NoDestructor<mediapipe::RegistrationToken>
    MessageRegistrationImpl<T>::registration(MessageHolderRegistry::Register(
-        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder,
-        __FILE__, __LINE__));
+        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder));

 // For non-Message payloads, this does nothing.
 template <typename T, typename Enable = void>
@@ -261,8 +261,8 @@ cc_library(
 )

 cc_library(
-    name = "opencv_highgui",
-    hdrs = ["opencv_highgui_inc.h"],
+    name = "opencv_photo",
+    hdrs = ["opencv_photo_inc.h"],
    deps = [
        ":opencv_core",
        "//third_party:opencv",
@@ -297,6 +297,15 @@ cc_library(
    ],
 )

+cc_library(
+    name = "opencv_highgui",
+    hdrs = ["opencv_highgui_inc.h"],
+    deps = [
+        ":opencv_core",
+        "//third_party:opencv",
+    ],
+)
+
 cc_library(
    name = "opencv_videoio",
    hdrs = ["opencv_videoio_inc.h"],
@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2023 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-#ifndef MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_
-#define MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_

 #include <opencv2/core/version.hpp>

@@ -25,4 +25,4 @@
 #include <opencv2/highgui.hpp>
 #endif

-#endif  // MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_
@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -0,0 +1,20 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+#define MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+
+#include "third_party/OpenCV/photo.hpp"
+
+#endif  // MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
@@ -1,4 +1,4 @@
-// Copyright 2019 The MediaPipe Authors.
+// Copyright 2022 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -273,8 +273,8 @@ absl::Status Scheduler::WaitForObservedOutput() {
 // Idleness requires:
 // 1. either the graph has no source nodes or all source nodes are closed, and
 // 2. no packets are added to graph input streams.
-// For simplicity, we only allow WaitUntilIdle() to be called on a graph with
-// no source nodes. (This is enforced by CalculatorGraph::WaitUntilIdle().)
+// For simplicity, we only fully support WaitUntilIdle() to be called on a graph
+// with no source nodes.
 // The application must ensure no other threads are adding packets to graph
 // input streams while a WaitUntilIdle() call is in progress.
 absl::Status Scheduler::WaitUntilIdle() {
@@ -48,6 +48,18 @@ class MuxInputStreamHandler : public InputStreamHandler {
      : InputStreamHandler(std::move(tag_map), cc_manager, options,
                           calculator_run_in_parallel) {}

+ private:
+  CollectionItemId GetControlStreamId() const {
+    return input_stream_managers_.EndId() - 1;
+  }
+  void RemoveOutdatedDataPackets(Timestamp timestamp) {
+    const CollectionItemId control_stream_id = GetControlStreamId();
+    for (CollectionItemId id = input_stream_managers_.BeginId();
+         id < control_stream_id; ++id) {
+      input_stream_managers_.Get(id)->ErasePacketsEarlierThan(timestamp);
+    }
+  }
+
 protected:
  // In MuxInputStreamHandler, a node is "ready" if:
  // - the control stream is done (need to call Close() in this case), or
@@ -58,9 +70,15 @@ class MuxInputStreamHandler : public InputStreamHandler {
    absl::MutexLock lock(&input_streams_mutex_);

    const auto& control_stream =
-        input_stream_managers_.Get(input_stream_managers_.EndId() - 1);
+        input_stream_managers_.Get(GetControlStreamId());
    bool empty;
    *min_stream_timestamp = control_stream->MinTimestampOrBound(&empty);
+
+    // Data streams may contain some outdated packets which failed to be popped
+    // out during "FillInputSet". (This handler doesn't sync input streams,
+    // hence "FillInputSet" can be triggerred before every input stream is
+    // filled with packets corresponding to the same timestamp.)
+    RemoveOutdatedDataPackets(*min_stream_timestamp);
    if (empty) {
      if (*min_stream_timestamp == Timestamp::Done()) {
        // Calculator is done if the control input stream is done.
@@ -78,11 +96,6 @@ class MuxInputStreamHandler : public InputStreamHandler {
    const auto& data_stream = input_stream_managers_.Get(
        input_stream_managers_.BeginId() + control_value);

-    // Data stream may contain some outdated packets which failed to be popped
-    // out during "FillInputSet". (This handler doesn't sync input streams,
-    // hence "FillInputSet" can be triggerred before every input stream is
-    // filled with packets corresponding to the same timestamp.)
-    data_stream->ErasePacketsEarlierThan(*min_stream_timestamp);
    Timestamp stream_timestamp = data_stream->MinTimestampOrBound(&empty);
    if (empty) {
      if (stream_timestamp <= *min_stream_timestamp) {
@@ -111,8 +124,7 @@ class MuxInputStreamHandler : public InputStreamHandler {
    CHECK(input_set);
    absl::MutexLock lock(&input_streams_mutex_);

-    const CollectionItemId control_stream_id =
-        input_stream_managers_.EndId() - 1;
+    const CollectionItemId control_stream_id = GetControlStreamId();
    auto& control_stream = input_stream_managers_.Get(control_stream_id);
    int num_packets_dropped = 0;
    bool stream_is_done = false;
@@ -140,15 +152,8 @@ class MuxInputStreamHandler : public InputStreamHandler {
    AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet),
                     stream_is_done);

-    // Discard old packets on other streams.
-    // Note that control_stream_id is the last valid id.
-    auto next_timestamp = input_timestamp.NextAllowedInStream();
-    for (CollectionItemId id = input_stream_managers_.BeginId();
-         id < control_stream_id; ++id) {
-      if (id == data_stream_id) continue;
-      auto& other_stream = input_stream_managers_.Get(id);
-      other_stream->ErasePacketsEarlierThan(next_timestamp);
-    }
+    // Discard old packets on data streams.
+    RemoveOutdatedDataPackets(input_timestamp.NextAllowedInStream());
  }

 private:
@@ -645,5 +645,41 @@ TEST(MuxInputStreamHandlerTest,
  MP_ASSERT_OK(graph.WaitUntilDone());
 }

+TEST(MuxInputStreamHandlerTest, RemovesUnusedDataStreamPackets) {
+  CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: "input0"
+        input_stream: "input1"
+        input_stream: "select"
+        node {
+          calculator: "MuxCalculator"
+          input_stream: "INPUT:0:input0"
+          input_stream: "INPUT:1:input1"
+          input_stream: "SELECT:select"
+          output_stream: "OUTPUT:output"
+          input_stream_handler { input_stream_handler: "MuxInputStreamHandler" }
+        }
+      )pb");
+  config.set_max_queue_size(1);
+  config.set_report_deadlock(true);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "select", MakePacket<int>(0).At(Timestamp(2))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input0", MakePacket<int>(1000).At(Timestamp(2))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Add two delayed packets to the deselected input. They should be discarded
+  // instead of triggering the deadlock detection (max_queue_size = 1).
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input1", MakePacket<int>(900).At(Timestamp(1))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input1", MakePacket<int>(900).At(Timestamp(2))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+}
+
 }  // namespace
 }  // namespace mediapipe
@@ -64,13 +64,13 @@ GraphRegistry::GraphRegistry(
 void GraphRegistry::Register(
    const std::string& type_name,
    std::function<std::unique_ptr<Subgraph>()> factory) {
-  local_factories_.Register(type_name, factory, __FILE__, __LINE__);
+  local_factories_.Register(type_name, factory);
 }

 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                             const CalculatorGraphConfig& config) {
-  Register(type_name, [config] {
+  local_factories_.Register(type_name, [config] {
    auto result = absl::make_unique<ProtoSubgraph>(config);
    return std::unique_ptr<Subgraph>(result.release());
  });
@@ -79,7 +79,7 @@ void GraphRegistry::Register(const std::string& type_name,
 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                             const CalculatorGraphTemplate& templ) {
-  Register(type_name, [templ] {
+  local_factories_.Register(type_name, [templ] {
    auto result = absl::make_unique<TemplateSubgraph>(templ);
    return std::unique_ptr<Subgraph>(result.release());
  });
@@ -131,6 +131,13 @@ Timestamp Timestamp::NextAllowedInStream() const {
  return *this + 1;
 }

+bool Timestamp::HasNextAllowedInStream() const {
+  if (*this >= Max() || *this == PreStream()) {
+    return false;
+  }
+  return true;
+}
+
 Timestamp Timestamp::PreviousAllowedInStream() const {
  if (*this <= Min() || *this == PostStream()) {
    // Indicates that no previous timestamps may occur.
@@ -186,6 +186,10 @@ class Timestamp {
  // CHECKs that this->IsAllowedInStream().
  Timestamp NextAllowedInStream() const;

+  // Returns true if there's a next timestamp in the range [Min .. Max] after
+  // this one.
+  bool HasNextAllowedInStream() const;
+
  // Returns the previous timestamp in the range [Min .. Max], or
  // Unstarted() if no Packets may preceed one with this timestamp.
  Timestamp PreviousAllowedInStream() const;
@@ -125,6 +125,22 @@ TEST(TimestampTest, NextAllowedInStream) {
            Timestamp::PostStream().NextAllowedInStream());
 }

+TEST(TimestampTest, HasNextAllowedInStream) {
+  EXPECT_TRUE(Timestamp::Min().HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Min() + 1).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(-1000).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(0).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(1000).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 2).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 1).HasNextAllowedInStream());
+
+  EXPECT_FALSE(Timestamp::PreStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Max().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::PostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::OneOverPostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Done().HasNextAllowedInStream());
+}
+
 TEST(TimestampTest, SpecialValueDifferences) {
  {  // Lower range
    const std::vector<Timestamp> timestamps = {
@@ -530,6 +530,7 @@ cc_library(
        "//mediapipe/framework/port:ret_check",
        "//mediapipe/framework/port:status",
        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_set",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/strings",
    ],
@@ -14,7 +14,7 @@

 """MediaPipe Task Library Helper Rules for iOS"""

-MPP_TASK_MINIMUM_OS_VERSION = "11.0"
+MPP_TASK_MINIMUM_OS_VERSION = "12.0"

 # When the static framework is built with bazel, the all header files are moved
 # to the "Headers" directory with no header path prefixes. This auxiliary rule
@@ -50,6 +50,7 @@ def mediapipe_proto_library_impl(
        def_cc_proto = True,
        def_py_proto = True,
        def_java_lite_proto = True,
+        def_kt_lite_proto = True,
        def_objc_proto = True,
        def_java_proto = True,
        def_jspb_proto = True,
@@ -72,6 +73,7 @@ def mediapipe_proto_library_impl(
      def_cc_proto: define the cc_proto_library target
      def_py_proto: define the py_proto_library target
      def_java_lite_proto: define the java_lite_proto_library target
+      def_kt_lite_proto: define the kt_lite_proto_library target
      def_objc_proto: define the objc_proto_library target
      def_java_proto: define the java_proto_library target
      def_jspb_proto: define the jspb_proto_library target
@@ -255,6 +257,7 @@ def mediapipe_proto_library(
        def_cc_proto = True,
        def_py_proto = True,
        def_java_lite_proto = True,
+        def_kt_lite_proto = True,
        def_portable_proto = True,  # @unused
        def_objc_proto = True,
        def_java_proto = True,
@@ -281,6 +284,7 @@ def mediapipe_proto_library(
      def_cc_proto: define the cc_proto_library target
      def_py_proto: define the py_proto_library target
      def_java_lite_proto: define the java_lite_proto_library target
+      def_kt_lite_proto: define the kt_lite_proto_library target
      def_portable_proto: ignored since portable protos are gone
      def_objc_proto: define the objc_proto_library target
      def_java_proto: define the java_proto_library target
@@ -304,6 +308,7 @@ def mediapipe_proto_library(
        def_cc_proto = def_cc_proto,
        def_py_proto = def_py_proto,
        def_java_lite_proto = def_java_lite_proto,
+        def_kt_lite_proto = def_kt_lite_proto,
        def_objc_proto = def_objc_proto,
        def_java_proto = def_java_proto,
        def_jspb_proto = def_jspb_proto,
@@ -334,6 +339,7 @@ def mediapipe_proto_library(
            def_cc_proto = def_cc_proto,
            def_py_proto = def_py_proto,
            def_java_lite_proto = def_java_lite_proto,
+            def_kt_lite_proto = def_kt_lite_proto,
            def_objc_proto = def_objc_proto,
            def_java_proto = def_java_proto,
            def_jspb_proto = def_jspb_proto,
@@ -20,6 +20,7 @@
 #include <utility>
 #include <vector>

+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
@@ -1430,10 +1431,10 @@ std::vector<const FieldDescriptor*> GetFields(const Message* src) {

 // Orders map entries in dst to match src.
 void OrderMapEntries(const Message* src, Message* dst,
-                     std::set<const Message*>* seen = nullptr) {
-  std::unique_ptr<std::set<const Message*>> seen_owner;
+                     absl::flat_hash_set<const Message*>* seen = nullptr) {
+  std::unique_ptr<absl::flat_hash_set<const Message*>> seen_owner;
  if (!seen) {
-    seen_owner = std::make_unique<std::set<const Message*>>();
+    seen_owner = std::make_unique<absl::flat_hash_set<const Message*>>();
    seen = seen_owner.get();
  }
  if (seen->count(src) > 0) {
@@ -1121,7 +1121,7 @@ objc_library(
    alwayslink = 1,
 )

-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"

 test_suite(
    name = "ios",
@@ -109,9 +109,8 @@ absl::Status GlContext::CreateContext(
  }
  MP_RETURN_IF_ERROR(status);

-  LOG(INFO) << "Successfully created a WebGL context with major version "
-            << gl_major_version_ << " and handle " << context_;
-
+  VLOG(1) << "Successfully created a WebGL context with major version "
+          << gl_major_version_ << " and handle " << context_;
  return absl::OkStatus();
 }

@@ -104,6 +104,7 @@ class GlScalerCalculator : public CalculatorBase {
  bool vertical_flip_output_;
  bool horizontal_flip_output_;
  FrameScaleMode scale_mode_ = FrameScaleMode::kStretch;
+  bool use_nearest_neighbor_interpolation_ = false;
 };
 REGISTER_CALCULATOR(GlScalerCalculator);

@@ -186,7 +187,8 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) {
    scale_mode_ =
        FrameScaleModeFromProto(options.scale_mode(), FrameScaleMode::kStretch);
  }
-
+  use_nearest_neighbor_interpolation_ =
+      options.use_nearest_neighbor_interpolation();
  if (HasTagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)) {
    const auto& dimensions =
        TagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)
@@ -297,6 +299,11 @@ absl::Status GlScalerCalculator::Process(CalculatorContext* cc) {
      glBindTexture(src2.target(), src2.name());
    }

+    if (use_nearest_neighbor_interpolation_) {
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    }
+
    MP_RETURN_IF_ERROR(renderer->GlRender(
        src1.width(), src1.height(), dst.width(), dst.height(), scale_mode_,
        rotation_, horizontal_flip_output_, vertical_flip_output_,
@@ -19,7 +19,7 @@ package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/gpu/scale_mode.proto";

-// Next id: 8.
+// Next id: 9.
 message GlScalerCalculatorOptions {
  extend CalculatorOptions {
    optional GlScalerCalculatorOptions ext = 166373014;
@@ -39,4 +39,7 @@ message GlScalerCalculatorOptions {
  // Flip the output texture horizontally. This is applied after rotation.
  optional bool flip_horizontal = 5;
  optional ScaleMode.Mode scale_mode = 6;
+  // Whether to use nearest neighbor interpolation. Default to use linear
+  // interpolation.
+  optional bool use_nearest_neighbor_interpolation = 8 [default = false];
 }
@@ -100,6 +100,10 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
               {GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1},
 #endif  // TARGET_OS_OSX
           }},
+          {GpuBufferFormat::kOneComponent8Alpha,
+           {
+               {GL_ALPHA, GL_ALPHA, GL_UNSIGNED_BYTE, 1},
+           }},
          {GpuBufferFormat::kOneComponent8Red,
           {
               {GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1},
@@ -221,6 +225,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
    case GpuBufferFormat::kRGBA32:
      // TODO: this likely maps to ImageFormat::SRGBA
    case GpuBufferFormat::kGrayHalf16:
+    case GpuBufferFormat::kOneComponent8Alpha:
    case GpuBufferFormat::kOneComponent8Red:
    case GpuBufferFormat::kTwoComponent8:
    case GpuBufferFormat::kTwoComponentHalf16:
@@ -43,6 +43,7 @@ enum class GpuBufferFormat : uint32_t {
  kGrayFloat32 = MEDIAPIPE_FOURCC('L', '0', '0', 'f'),
  kGrayHalf16 = MEDIAPIPE_FOURCC('L', '0', '0', 'h'),
  kOneComponent8 = MEDIAPIPE_FOURCC('L', '0', '0', '8'),
+  kOneComponent8Alpha = MEDIAPIPE_FOURCC('A', '0', '0', '8'),
  kOneComponent8Red = MEDIAPIPE_FOURCC('R', '0', '0', '8'),
  kTwoComponent8 = MEDIAPIPE_FOURCC('2', 'C', '0', '8'),
  kTwoComponentHalf16 = MEDIAPIPE_FOURCC('2', 'C', '0', 'h'),
@@ -101,6 +102,7 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
      return kCVPixelFormatType_OneComponent32Float;
    case GpuBufferFormat::kOneComponent8:
      return kCVPixelFormatType_OneComponent8;
+    case GpuBufferFormat::kOneComponent8Alpha:
    case GpuBufferFormat::kOneComponent8Red:
      return -1;
    case GpuBufferFormat::kTwoComponent8:
@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
 import javax.microedition.khronos.egl.EGLConfig;
 import javax.microedition.khronos.opengles.GL10;

@@ -303,7 +304,7 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
  }

  // Use this when the texture is not a SurfaceTexture.
-  public void setNextFrame(TextureFrame frame) {
+  public void setNextFrame(@Nullable TextureFrame frame) {
    if (surfaceTexture != null) {
      Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */);
    }
@@ -78,17 +78,21 @@ public class AppTextureFrame implements TextureFrame {
   * Use {@link waitUntilReleasedWithGpuSync} whenever possible.
   */
  public void waitUntilReleased() throws InterruptedException {
+    GlSyncToken tokenToRelease = null;
    synchronized (this) {
      while (inUse && releaseSyncToken == null) {
        wait();
      }
      if (releaseSyncToken != null) {
-        releaseSyncToken.waitOnCpu();
-        releaseSyncToken.release();
+        tokenToRelease = releaseSyncToken;
        inUse = false;
        releaseSyncToken = null;
      }
    }
+    if (tokenToRelease != null) {
+      tokenToRelease.waitOnCpu();
+      tokenToRelease.release();
+    }
  }

  /**
@@ -98,17 +102,21 @@ public class AppTextureFrame implements TextureFrame {
   * TextureFrame.
   */
  public void waitUntilReleasedWithGpuSync() throws InterruptedException {
+    GlSyncToken tokenToRelease = null;
    synchronized (this) {
      while (inUse && releaseSyncToken == null) {
        wait();
      }
      if (releaseSyncToken != null) {
-        releaseSyncToken.waitOnGpu();
-        releaseSyncToken.release();
+        tokenToRelease = releaseSyncToken;
        inUse = false;
        releaseSyncToken = null;
      }
    }
+    if (tokenToRelease != null) {
+      tokenToRelease.waitOnGpu();
+      tokenToRelease.release();
+    }
  }

  /**
@@ -50,7 +50,6 @@ android_library(
        "MediaPipeRunner.java",
    ],
    visibility = [
-        "//java/com/google/android/libraries/camera/effects:__subpackages__",
        "//mediapipe/java/com/google/mediapipe:__subpackages__",
    ],
    exports = [
@@ -239,7 +239,7 @@ public final class PacketGetter {

  /**
   * Assign the native image buffer array in given ByteBuffer array. It assumes given ByteBuffer
-   * array has the the same size of image list packet, and assumes the output buffer stores pixels
+   * array has the same size of image list packet, and assumes the output buffer stores pixels
   * contiguously. It returns false if this assumption does not hold.
   *
   * <p>If deepCopy is true, it assumes the given buffersArray has allocated the required size of
@@ -67,6 +67,7 @@ public class ExternalTextureRenderer {
  private float[] textureTransformMatrix = new float[16];
  private boolean flipY;
  private int rotation = Surface.ROTATION_0;
+  private boolean doExplicitCpuSync = true;

  /** Call this to setup the shader program before rendering. */
  public void setup() {
@@ -101,6 +102,14 @@ public class ExternalTextureRenderer {
    this.rotation = rotation;
  }

+  /**
+   * Configures whether the renderer should do an explicit CPU synchronization using glFinish upon
+   * each {@link #render} call. Defaults to true.
+   */
+  public void setDoExplicitCpuSync(boolean doExplicitCpuSync) {
+    this.doExplicitCpuSync = doExplicitCpuSync;
+  }
+
  /**
   * Renders the surfaceTexture to the framebuffer with optional vertical flip.
   *
@@ -150,8 +159,11 @@ public class ExternalTextureRenderer {
    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
    ShaderUtil.checkGlError("glBindTexture");

-    // TODO: add sync and go back to glFlush()
-    GLES20.glFinish();
+    if (doExplicitCpuSync) {
+
+      // TODO: add sync and go back to glFlush()
+      GLES20.glFinish();
+    }
  }

  /**
@@ -24,6 +24,7 @@ package_group(
 package_group(
    name = "1p_client",
    packages = [
+        "//cloud/ml/applications/vision/model_garden/model_oss/mediapipe/...",
        "//research/privacy/learning/fl_eval/pcvr/...",
    ],
 )
@@ -14,7 +14,10 @@

 # Placeholder for internal Python strict library and test compatibility macro.

-package(default_visibility = ["//mediapipe:__subpackages__"])
+package(default_visibility = [
+    "//cloud/ml/applications/vision/model_garden/model_oss/mediapipe:__subpackages__",
+    "//mediapipe:__subpackages__",
+])

 licenses(["notice"])

@@ -57,3 +57,14 @@ py_test(
    srcs = ["classification_dataset_test.py"],
    deps = [":classification_dataset"],
 )
+
+py_library(
+    name = "cache_files",
+    srcs = ["cache_files.py"],
+)
+
+py_test(
+    name = "cache_files_test",
+    srcs = ["cache_files_test.py"],
+    deps = [":cache_files"],
+)
--- a/Show More
+++ b/Show More