Bump protobufjs from 7.1.2 to 7.2.4

Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.1.2 to 7.2.4.
- [Release notes](https://github.com/protobufjs/protobuf.js/releases)
- [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md)
- [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.1.2...protobufjs-v7.2.4)

---
updated-dependencies:
- dependency-name: protobufjs
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <[email protected]>

README.md

@@ -1,17 +1,16 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
 title: Home
 nav_order: 1
 ---
 
 ----
 
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
 [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
 as the primary developer documentation site for MediaPipe as of April 3, 2023.*
 
-*This notice and web page will be removed on June 1, 2023.*
-
 ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
 
 **Attention**: MediaPipe Solutions Preview is an early release. [Learn

WORKSPACE

@@ -45,12 +45,13 @@ http_archive(
 )
 
 http_archive(
-   name = "rules_foreign_cc",
+    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-0.1.0",
+    sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
+    strip_prefix = "rules_foreign_cc-0.9.0",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
 )
 
-load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
 rules_foreign_cc_dependencies()
 
@@ -156,22 +157,22 @@ http_archive(
 # 2020-08-21
 http_archive(
     name = "com_github_glog_glog",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
+    strip_prefix = "glog-0.6.0",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
     urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/v0.6.0.tar.gz",
     ],
 )
 http_archive(
     name = "com_github_glog_glog_no_gflags",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
+    strip_prefix = "glog-0.6.0",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
     build_file = "@//third_party:glog_no_gflags.BUILD",
     urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/v0.6.0.tar.gz",
     ],
     patches = [
-        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
+        "@//third_party:com_github_glog_glog.diff",
     ],
     patch_args = [
         "-p1",
@@ -266,10 +267,10 @@ http_archive(
 
 http_archive(
     name = "com_googlesource_code_re2",
-    sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
+    sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
-    strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
+    strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
     urls = [
-        "https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
+        "https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
     ],
 )
 
@@ -484,9 +485,10 @@ http_archive(
 )
 
 # TensorFlow repo should always go after the other external dependencies.
-# TF on 2023-04-12.
+# TF on 2023-06-13.
-_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
+_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
-_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
+# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
+_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
 http_archive(
     name = "org_tensorflow",
     urls = [

docs/index.md

@@ -1,17 +1,16 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
 title: Home
 nav_order: 1
 ---
 
 ----
 
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
 [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
 as the primary developer documentation site for MediaPipe as of April 3, 2023.*
 
-*This notice and web page will be removed on June 1, 2023.*
-
 ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
 
 **Attention**: MediaPipe Solutions Preview is an early release. [Learn

docs/solutions/face_detection.md

@@ -20,9 +20,9 @@ nav_order: 1
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
 site.*
 
 ----

docs/solutions/face_mesh.md

@@ -20,9 +20,9 @@ nav_order: 2
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
 site.*
 
 ----

docs/solutions/iris.md

@@ -20,9 +20,9 @@ nav_order: 3
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
 site.*
 
 ----

docs/solutions/pose.md

@@ -22,9 +22,9 @@ nav_order: 5
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
 site.*
 
 ----

docs/solutions/pose_classification.md

@@ -21,7 +21,7 @@ nav_order: 1
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
 [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
 site.*

docs/solutions/solutions.md

@@ -1,5 +1,6 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe/solutions/guide#legacy
 title: MediaPipe Legacy Solutions
 nav_order: 3
 has_children: true
@@ -13,8 +14,7 @@ has_toc: false
 {:toc}
 ---
 
-**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
+**Attention:** *We have ended support for
-ended support for
 [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
 as of March 1, 2023. All other
 [MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
 Solutions at:
 [https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
 
-*This notice and web page will be removed on June 1, 2023.*
-
-----
-
-<br><br><br><br><br><br><br><br><br><br>
-<br><br><br><br><br><br><br><br><br><br>
-<br><br><br><br><br><br><br><br><br><br>
-
 ----
 
 MediaPipe offers open source cross-platform, customizable ML solutions for live

mediapipe/calculators/audio/BUILD

@@ -219,12 +219,10 @@ cc_library(
     deps = [
         ":time_series_framer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:integral_types",
-        "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@eigen_archive//:eigen3",
@@ -319,6 +317,20 @@ cc_test(
     ],
 )
 
+cc_binary(
+    name = "time_series_framer_calculator_benchmark",
+    srcs = ["time_series_framer_calculator_benchmark.cc"],
+    deps = [
+        ":time_series_framer_calculator",
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "@com_google_benchmark//:benchmark",
+    ],
+)
+
 cc_test(
     name = "time_series_framer_calculator_test",
     srcs = ["time_series_framer_calculator_test.cc"],

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 // Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
 const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
 
+namespace {
+std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
+    const SpectrogramCalculatorOptions::WindowType window_type) {
+  switch (window_type) {
+    // The cosine window and square root of Hann are equivalent.
+    case SpectrogramCalculatorOptions::COSINE:
+    case SpectrogramCalculatorOptions::SQRT_HANN:
+      return std::make_unique<audio_dsp::CosineWindow>();
+    case SpectrogramCalculatorOptions::HANN:
+      return std::make_unique<audio_dsp::HannWindow>();
+    case SpectrogramCalculatorOptions::HAMMING:
+      return std::make_unique<audio_dsp::HammingWindow>();
+  }
+  return nullptr;
+}
+}  // namespace
+
 absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
   SpectrogramCalculatorOptions spectrogram_options =
       cc->Options<SpectrogramCalculatorOptions>();
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
 
   output_scale_ = spectrogram_options.output_scale();
 
-  std::vector<double> window;
+  auto window_fun = MakeWindowFun(spectrogram_options.window_type());
-  switch (spectrogram_options.window_type()) {
+  if (window_fun == nullptr) {
-    case SpectrogramCalculatorOptions::COSINE:
+    return absl::Status(absl::StatusCode::kInvalidArgument,
-      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                        absl::StrCat("Invalid window type ",
-                                                   &window);
+                                     spectrogram_options.window_type()));
-      break;
-    case SpectrogramCalculatorOptions::HANN:
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      break;
-    case SpectrogramCalculatorOptions::HAMMING:
-      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                    &window);
-      break;
-    case SpectrogramCalculatorOptions::SQRT_HANN: {
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      absl::c_transform(window, window.begin(),
-                        [](double x) { return std::sqrt(x); });
-      break;
-    }
   }
+  std::vector<double> window;
+  window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
 
   // Propagate settings down to the actual Spectrogram object.
   spectrogram_generators_.clear();

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
     HANN = 0;
     HAMMING = 1;
     COSINE = 2;
-    SQRT_HANN = 4;
+    SQRT_HANN = 4;  // Alias of COSINE.
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -15,9 +15,7 @@
 // Defines TimeSeriesFramerCalculator.
 #include <math.h>
 
-#include <deque>
+#include <vector>
-#include <memory>
-#include <string>
 
 #include "Eigen/Core"
 #include "audio/dsp/window_functions.h"
@@ -25,9 +23,8 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/util/time_series_util.h"
 
 namespace mediapipe {
@@ -88,11 +85,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  // Adds input data to the internal buffer.
-  void EnqueueInput(CalculatorContext* cc);
-  // Constructs and emits framed output packets.
-  void FrameOutput(CalculatorContext* cc);
-
   Timestamp CurrentOutputTimestamp() {
     if (use_local_timestamp_) {
       return current_timestamp_;
@@ -106,14 +98,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
                  Timestamp::kTimestampUnitsPerSecond);
   }
 
-  // Returns the timestamp of a sample on a base, which is usually the time
-  // stamp of a packet.
-  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
-                                   int64_t number_of_samples) {
-    return timestamp_base + round(number_of_samples / sample_rate_ *
-                                  Timestamp::kTimestampUnitsPerSecond);
-  }
-
   // The number of input samples to advance after the current output frame is
   // emitted.
   int next_frame_step_samples() const {
@@ -142,61 +126,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   Timestamp initial_input_timestamp_;
   // The current timestamp is updated along with the incoming packets.
   Timestamp current_timestamp_;
-  int num_channels_;
 
-  // Each entry in this deque consists of a single sample, i.e. a
+  // Samples are buffered in a vector of sample blocks.
-  // single column vector, and its timestamp.
+  class SampleBlockBuffer {
-  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
+   public:
+    // Initializes the buffer.
+    void Init(double sample_rate, int num_channels) {
+      ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
+      num_channels_ = num_channels;
+      num_samples_ = 0;
+      first_block_offset_ = 0;
+    }
+
+    // Number of channels, equal to the number of rows in each Matrix.
+    int num_channels() const { return num_channels_; }
+    // Total number of available samples over all blocks.
+    int num_samples() const { return num_samples_; }
+
+    // Pushes a new block of samples on the back of the buffer with `timestamp`
+    // being the input timestamp of the packet containing the Matrix.
+    void Push(const Matrix& samples, Timestamp timestamp);
+    // Copies `count` samples from the front of the buffer. If there are fewer
+    // samples than this, the result is zero padded to have `count` samples.
+    // The timestamp of the last copied sample is written to *last_timestamp.
+    // This output is used below to update `current_timestamp_`, which is only
+    // used when `use_local_timestamp` is true.
+    Matrix CopySamples(int count, Timestamp* last_timestamp) const;
+    // Drops `count` samples from the front of the buffer. If `count` exceeds
+    // `num_samples()`, the buffer is emptied.  Returns how many samples were
+    // dropped.
+    int DropSamples(int count);
+
+   private:
+    struct Block {
+      // Matrix of num_channels rows by num_samples columns, a block of possibly
+      // multiple samples.
+      Matrix samples;
+      // Timestamp of the first sample in the Block. This comes from the input
+      // packet's timestamp that contains this Matrix.
+      Timestamp timestamp;
+
+      Block() : timestamp(Timestamp::Unstarted()) {}
+      Block(const Matrix& samples, Timestamp timestamp)
+          : samples(samples), timestamp(timestamp) {}
+      int num_samples() const { return samples.cols(); }
+    };
+    std::vector<Block> blocks_;
+    // Number of timestamp units per sample. Used to compute timestamps as
+    // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
+    double ts_units_per_sample_;
+    // Number of rows in each Matrix.
+    int num_channels_;
+    // The total number of samples over all blocks, equal to
+    // (sum_i blocks_[i].num_samples()) - first_block_offset_.
+    int num_samples_;
+    // The number of samples in the first block that have been discarded. This
+    // way we can cheaply represent "partially discarding" a block.
+    int first_block_offset_;
+  } sample_buffer_;
 
   bool use_window_;
-  Matrix window_;
+  Eigen::RowVectorXf window_;
 
   bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
 
-void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
+void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
-  const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
+                                                         Timestamp timestamp) {
-
+  num_samples_ += samples.cols();
-  for (int i = 0; i < input_frame.cols(); ++i) {
+  blocks_.emplace_back(samples, timestamp);
-    sample_buffer_.emplace_back(std::make_pair(
-        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
-  }
 }
 
-void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
+Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
-  while (sample_buffer_.size() >=
+    int count, Timestamp* last_timestamp) const {
+  Matrix copied(num_channels_, count);
+
+  if (!blocks_.empty()) {
+    int num_copied = 0;
+    // First block has an offset for samples that have been discarded.
+    int offset = first_block_offset_;
+    int n;
+    Timestamp last_block_ts;
+    int last_sample_index;
+
+    for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
+      n = std::min(it->num_samples() - offset, count);
+      // Copy `n` samples from the next block.
+      copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
+      count -= n;
+      num_copied += n;
+      last_block_ts = it->timestamp;
+      last_sample_index = offset + n - 1;
+      offset = 0;  // No samples have been discarded in subsequent blocks.
+    }
+
+    // Compute the timestamp of the last copied sample.
+    *last_timestamp =
+        last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
+  }
+
+  if (count > 0) {
+    copied.rightCols(count).setZero();  // Zero pad if needed.
+  }
+
+  return copied;
+}
+
+int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
+  if (blocks_.empty()) {
+    return 0;
+  }
+
+  auto block_it = blocks_.begin();
+  if (first_block_offset_ + count < block_it->num_samples()) {
+    // `count` is less than the remaining samples in the first block.
+    first_block_offset_ += count;
+    num_samples_ -= count;
+    return count;
+  }
+
+  int num_samples_dropped = block_it->num_samples() - first_block_offset_;
+  count -= num_samples_dropped;
+  first_block_offset_ = 0;
+
+  for (++block_it; block_it != blocks_.end(); ++block_it) {
+    if (block_it->num_samples() > count) {
+      break;
+    }
+    num_samples_dropped += block_it->num_samples();
+    count -= block_it->num_samples();
+  }
+
+  blocks_.erase(blocks_.begin(), block_it);  // Drop whole blocks.
+  if (!blocks_.empty()) {
+    first_block_offset_ = count;  // Drop part of the next block.
+    num_samples_dropped += count;
+  }
+
+  num_samples_ -= num_samples_dropped;
+  return num_samples_dropped;
+}
+
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
+  }
+
+  // Add input data to the internal buffer.
+  sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
+                      cc->InputTimestamp());
+
+  // Construct and emit framed output packets.
+  while (sample_buffer_.num_samples() >=
          frame_duration_samples_ + samples_still_to_drop_) {
-    while (samples_still_to_drop_ > 0) {
+    sample_buffer_.DropSamples(samples_still_to_drop_);
-      sample_buffer_.pop_front();
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
-      --samples_still_to_drop_;
+                                                     &current_timestamp_);
-    }
     const int frame_step_samples = next_frame_step_samples();
-    std::unique_ptr<Matrix> output_frame(
+    samples_still_to_drop_ = frame_step_samples;
-        new Matrix(num_channels_, frame_duration_samples_));
-    for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
-         ++i) {
-      output_frame->col(i) = sample_buffer_.front().first;
-      current_timestamp_ = sample_buffer_.front().second;
-      sample_buffer_.pop_front();
-    }
-    const int frame_overlap_samples =
-        frame_duration_samples_ - frame_step_samples;
-    if (frame_overlap_samples > 0) {
-      for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
-        current_timestamp_ = sample_buffer_[i].second;
-      }
-    } else {
-      samples_still_to_drop_ = -frame_overlap_samples;
-    }
 
     if (use_window_) {
-      *output_frame = (output_frame->array() * window_.array()).matrix();
+      // Apply the window to each row of output_frame.
+      output_frame.array().rowwise() *= window_.array();
     }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
-                               CurrentOutputTimestamp());
+                                         .At(CurrentOutputTimestamp()));
     ++cumulative_output_frames_;
     cumulative_completed_samples_ += frame_step_samples;
   }
@@ -206,35 +303,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
     // fact to enable packet queueing optimizations.
     cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
   }
-}
-
-absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
-  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
-    initial_input_timestamp_ = cc->InputTimestamp();
-    current_timestamp_ = initial_input_timestamp_;
-  }
-
-  EnqueueInput(cc);
-  FrameOutput(cc);
 
   return absl::OkStatus();
 }
 
 absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
-  while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
+  sample_buffer_.DropSamples(samples_still_to_drop_);
-    sample_buffer_.pop_front();
-    --samples_still_to_drop_;
-  }
-  if (!sample_buffer_.empty() && pad_final_packet_) {
-    std::unique_ptr<Matrix> output_frame(new Matrix);
-    output_frame->setZero(num_channels_, frame_duration_samples_);
-    for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i].first;
-      current_timestamp_ = sample_buffer_[i].second;
-    }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+  if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
-                               CurrentOutputTimestamp());
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+                                                     &current_timestamp_);
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
+                                         .At(CurrentOutputTimestamp()));
   }
 
   return absl::OkStatus();
@@ -258,7 +338,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
       cc->Inputs().Index(0).Header(), &input_header));
 
   sample_rate_ = input_header.sample_rate();
-  num_channels_ = input_header.num_channels();
+  sample_buffer_.Init(sample_rate_, input_header.num_channels());
   frame_duration_samples_ = time_series_util::SecondsToSamples(
       framer_options.frame_duration_seconds(), sample_rate_);
   RET_CHECK_GT(frame_duration_samples_, 0)
@@ -312,9 +392,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   }
 
   if (use_window_) {
-    window_ = Matrix::Ones(num_channels_, 1) *
+    window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
-              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
+                                             frame_duration_samples_)
-                                          frame_duration_samples_)
                   .cast<float>();
   }
   use_local_timestamp_ = framer_options.use_local_timestamp();

mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc

@@ -0,0 +1,92 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Benchmark for TimeSeriesFramerCalculator.
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/packet.h"
+
+using ::mediapipe::Matrix;
+
+void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
+  constexpr float kSampleRate = 32000.0;
+  constexpr int kNumChannels = 2;
+  constexpr int kFrameDurationSeconds = 5.0;
+  std::mt19937 rng(0 /*seed*/);
+  // Input around a half second's worth of samples at a time.
+  std::uniform_int_distribution<int> input_size_dist(15000, 17000);
+  // Generate a pool of random blocks of samples up front.
+  std::vector<Matrix> sample_pool;
+  sample_pool.reserve(20);
+  for (int i = 0; i < 20; ++i) {
+    sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
+  }
+  std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
+
+  mediapipe::CalculatorGraphConfig config;
+  config.add_input_stream("input");
+  config.add_output_stream("output");
+  auto* node = config.add_node();
+  node->set_calculator("TimeSeriesFramerCalculator");
+  node->add_input_stream("input");
+  node->add_output_stream("output");
+  mediapipe::TimeSeriesFramerCalculatorOptions* options =
+      node->mutable_options()->MutableExtension(
+          mediapipe::TimeSeriesFramerCalculatorOptions::ext);
+  options->set_frame_duration_seconds(kFrameDurationSeconds);
+
+  for (auto _ : state) {
+    state.PauseTiming();  // Pause benchmark timing.
+
+    // Prepare input packets of random blocks of samples.
+    std::vector<mediapipe::Packet> input_packets;
+    input_packets.reserve(32);
+    float t = 0;
+    for (int i = 0; i < 32; ++i) {
+      auto samples =
+          std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
+      const int num_samples = samples->cols();
+      input_packets.push_back(mediapipe::Adopt(samples.release())
+                                  .At(mediapipe::Timestamp::FromSeconds(t)));
+      t += num_samples / kSampleRate;
+    }
+    // Initialize graph.
+    mediapipe::CalculatorGraph graph;
+    CHECK_OK(graph.Initialize(config));
+    // Prepare input header.
+    auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
+    header->set_sample_rate(kSampleRate);
+    header->set_num_channels(kNumChannels);
+
+    state.ResumeTiming();  // Resume benchmark timing.
+
+    CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
+    for (auto& packet : input_packets) {
+      CHECK_OK(graph.AddPacketToInputStream("input", packet));
+    }
+    CHECK(!graph.HasError());
+    CHECK_OK(graph.CloseAllInputStreams());
+    CHECK_OK(graph.WaitUntilIdle());
+  }
+}
+BENCHMARK(BM_TimeSeriesFramerCalculator);
+
+BENCHMARK_MAIN();

mediapipe/calculators/core/BUILD

@@ -117,6 +117,7 @@ mediapipe_proto_library(
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework/formats:classification_proto",
         "//mediapipe/framework/formats:landmark_proto",
+        "//mediapipe/framework/formats:matrix_data_proto",
         "//mediapipe/framework/formats:time_series_header_proto",
     ],
 )
@@ -192,17 +193,19 @@ cc_library(
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/gpu:gpu_buffer",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
     ],
     alwayslink = 1,
 )
@@ -215,18 +218,18 @@ cc_library(
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
+        "//mediapipe/gpu:gpu_buffer",
         "//mediapipe/util:render_data_cc_proto",
+        "@com_google_absl//absl/status",
         "@org_tensorflow//tensorflow/lite:framework",
     ],
     alwayslink = 1,
@@ -287,6 +290,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/api2:port",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:integral_types",
@@ -295,8 +299,7 @@ cc_library(
         "//mediapipe/util:render_data_cc_proto",
         "@org_tensorflow//tensorflow/lite:framework",
     ] + select({
-        "//mediapipe/gpu:disable_gpu": [],
+        ":ios_or_disable_gpu": [],
-        "//mediapipe:ios": [],
         "//conditions:default": [
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
         ],
@@ -904,6 +907,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
@@ -1136,6 +1140,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
@@ -1164,6 +1169,7 @@ cc_library(
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -1238,6 +1244,7 @@ cc_library(
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ],

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -17,10 +17,13 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
 
 namespace mediapipe {
 
@@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
 typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
 REGISTER_CALCULATOR(BeginLoopTensorCalculator);
 
+// A calculator to process std::vector<mediapipe::ImageFrame>.
+typedef BeginLoopCalculator<std::vector<ImageFrame>>
+    BeginLoopImageFrameCalculator;
+REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
+
+// A calculator to process std::vector<mediapipe::GpuBuffer>.
+typedef BeginLoopCalculator<std::vector<GpuBuffer>>
+    BeginLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
+
+// A calculator to process std::vector<mediapipe::Image>.
+typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
+REGISTER_CALCULATOR(BeginLoopImageCalculator);
+
+// A calculator to process std::vector<float>.
+typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
+REGISTER_CALCULATOR(BeginLoopFloatCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.h

@@ -15,47 +15,57 @@
 #ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
 
+#include "absl/status/status.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/status_macros.h"
 
 namespace mediapipe {
 
 // Calculator for implementing loops on iterable collections inside a MediaPipe
-// graph.
+// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
+// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
+// First, instantiate specializations in the loop calculators' implementations
+// if missing:
+//   BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
+//   EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
+// Then, the following graph transforms an item of type InputIterT to an
+// OutputIterT by applying InputToOutputConverter to every element:
 //
-// It is designed to be used like:
+// node {                                        # Type        @timestamp
-//
+//   calculator:    "BeginLoopInputTCalculator"
-// node {
+//   input_stream:  "ITERABLE:input_iterable"    # InputIterT  @iterable_ts
-//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "CLONE:extra_input"          # ExtraT      @extra_ts
-//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_iterator"        # InputT      @loop_internal_ts
-//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "CLONE:cloned_extra_input"   # ExtraT      @loop_internal_ts
-//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "BATCH_END:iterable_ts"      # Timestamp   @loop_internal_ts
 // }
 //
 // node {
-//   calculator:    "ElementToBlaConverterSubgraph"
+//   calculator:    "InputToOutputConverter"
-//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   input_stream:  "INPUT:input_iterator"       # InputT      @loop_internal_ts
-//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+//   input_stream:  "EXTRA:cloned_extra_input"   # ExtraT      @loop_internal_ts
+//   output_stream: "OUTPUT:output_iterator"     # OutputT     @loop_internal_ts
 // }
 //
 // node {
-//   calculator:    "EndLoopWithOutputCalculator"
+//   calculator:    "EndLoopOutputTCalculator"
-//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "ITEM:output_iterator"       # OutputT     @loop_internal_ts
-//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   input_stream:  "BATCH_END:iterable_ts"      # Timestamp   @loop_internal_ts
-//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
+//   output_stream: "ITERABLE:output_iterable"   # OutputIterT @iterable_ts
 // }
 //
+// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
+// The output packets of this calculator are part of the loop body and have
+// loop-internal timestamps that are unrelated to the input iterator timestamp.
+//
 // Input streams tagged with "CLONE" are cloned to the corresponding output
-// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
+// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
-// can run multiple times, once per element in the "ITERABLE" for each pakcet
+// sub-graph can run multiple times, once per element in the "ITERABLE" for each
-// clone of the packets in the "CLONE" input streams.
+// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
+// inputs as loop-wide constants.
 template <typename IterableT>
 class BeginLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
 typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
 
+typedef ConcatenateVectorCalculator<std::string>
+    ConcatenateStringVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"
@@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
     ConcatenateRenderDataVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
 
+typedef ConcatenateVectorCalculator<mediapipe::Image>
+    ConcatenateImageVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -30,13 +30,15 @@ namespace mediapipe {
 typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
 
-void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
+template <typename T>
+void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
                     CalculatorRunner* runner) {
   runner->MutableInputs()->Index(index).packets.push_back(
-      MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
+      MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
 }
 
-void AddInputVectors(const std::vector<std::vector<int>>& inputs,
+template <typename T>
+void AddInputVectors(const std::vector<std::vector<T>>& inputs,
                      int64_t timestamp, CalculatorRunner* runner) {
   for (int i = 0; i < inputs.size(); ++i) {
     AddInputVector(i, inputs[i], timestamp, runner);
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
   EXPECT_EQ(0, outputs.size());
 }
 
+TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("ConcatenateStringVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<std::string>> inputs = {
+      {"a", "b"}, {"c"}, {"d", "e", "f"}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
+}
+
 typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
     TestConcatenateUniqueIntPtrCalculator;
 MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
         packet.Set<LandmarkList>();
       } else if (packet_options.has_double_value()) {
         packet.Set<double>();
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set<MatrixData>();
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set<TimeSeriesHeader>();
+      } else if (packet_options.has_int64_value()) {
+        packet.Set<int64_t>();
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
             MakePacket<LandmarkList>(packet_options.landmark_list_value()));
       } else if (packet_options.has_double_value()) {
         packet.Set(MakePacket<double>(packet_options.double_value()));
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set(MakePacket<TimeSeriesHeader>(
             packet_options.time_series_header_value()));
+      } else if (packet_options.has_int64_value()) {
+        packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -19,6 +19,7 @@ package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
 import "mediapipe/framework/formats/landmark.proto";
+import "mediapipe/framework/formats/matrix_data.proto";
 import "mediapipe/framework/formats/time_series_header.proto";
 
 message ConstantSidePacketCalculatorOptions {
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
   message ConstantSidePacket {
     oneof value {
       int32 int_value = 1;
+      uint64 uint64_value = 5;
+      int64 int64_value = 11;
       float float_value = 2;
+      double double_value = 9;
       bool bool_value = 3;
       string string_value = 4;
-      uint64 uint64_value = 5;
       ClassificationList classification_list_value = 6;
       LandmarkList landmark_list_value = 7;
-      double double_value = 9;
       TimeSeriesHeader time_series_header_value = 10;
+      MatrixData matrix_data_value = 12;
     }
   }
 

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
 #include <string>
 
 #include "absl/strings/string_view.h"
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
   DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
   DoTestSingleSidePacket("{ bool_value: true }", true);
   DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+  DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
 }
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {

mediapipe/calculators/core/end_loop_calculator.cc

@@ -19,10 +19,12 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/util/render_data.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
@@ -68,8 +70,18 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
 typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
 REGISTER_CALCULATOR(EndLoopTensorCalculator);
 
+typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
+REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
+
+typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
+
 typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
     EndLoopImageCalculator;
 REGISTER_CALCULATOR(EndLoopImageCalculator);
 
+typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
+    EndLoopAffineMatrixCalculator;
+REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.h

@@ -17,13 +17,11 @@
 
 #include <type_traits>
 
+#include "absl/status/status.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
 
@@ -33,27 +31,7 @@ namespace mediapipe {
 // from the "BATCH_END" tagged input stream, it emits the aggregated results
 // at the original timestamp contained in the "BATCH_END" input stream.
 //
-// It is designed to be used like:
+// See BeginLoopCalculator for a usage example.
-//
-// node {
-//   calculator:    "BeginLoopWithIterableCalculator"
-//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
-//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
-//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-// }
-//
-// node {
-//   calculator:    "ElementToBlaConverterSubgraph"
-//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
-//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
-// }
-//
-// node {
-//   calculator:    "EndLoopWithOutputCalculator"
-//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
-//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
-// }
 template <typename IterableT>
 class EndLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;
@@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
       }
       // Try to consume the item and move it into the collection. If the items
       // are not consumable, then try to copy them instead. If the items are
-      // not copiable, then an error will be returned.
+      // not copyable, then an error will be returned.
       auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
       if (item_ptr_or.ok()) {
         input_stream_collection_->push_back(std::move(*item_ptr_or.value()));

mediapipe/calculators/core/get_vector_item_calculator.cc

@@ -17,6 +17,7 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
 
 namespace mediapipe {
 namespace api2 {
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
     GetVectorItemCalculator<mediapipe::Detection>;
 REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
 
+using GetNormalizedRectVectorItemCalculator =
+    GetVectorItemCalculator<NormalizedRect>;
+REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
+
+using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
+REGISTER_CALCULATOR(GetRectVectorItemCalculator);
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
           // However, LOOP packet is empty.
           kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
         } else {
-          kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          // Avoids sending leftovers to a stream that's already closed.
+          if (!kPrevLoop(cc).IsClosed()) {
+            kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          }
         }
         loop_packets_.pop_front();
         main_packet_specs_.pop_front();

mediapipe/calculators/core/split_vector_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
 typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
 REGISTER_CALCULATOR(SplitFloatVectorCalculator);
 
+typedef SplitVectorCalculator<mediapipe::Image, false>
+    SplitImageVectorCalculator;
+REGISTER_CALCULATOR(SplitImageVectorCalculator);
+
+typedef SplitVectorCalculator<std::array<float, 16>, false>
+    SplitAffineMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/stream_to_side_packet_calculator.cc

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // A calculator that takes a packet of an input stream and converts it to an
 // output side packet. This calculator only works under the assumption that the
@@ -28,21 +30,21 @@ namespace mediapipe {
 //   input_stream: "stream"
 //   output_side_packet: "side_packet"
 // }
-class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+class StreamToSidePacketCalculator : public Node {
  public:
-  static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
+  static constexpr Input<AnyType>::Optional kIn{""};
-    cc->Inputs().Index(0).SetAny();
+  static constexpr SideOutput<SameType<kIn>> kOut{""};
-    cc->OutputSidePackets().Index(0).SetAny();
+
-    return absl::OkStatus();
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
-  }
 
   absl::Status Process(mediapipe::CalculatorContext* cc) override {
-    mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
+    kOut(cc).Set(
-    cc->OutputSidePackets().Index(0).Set(
+        kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
-        packet.At(mediapipe::Timestamp::Unset()));
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(StreamToSidePacketCalculator);
 
+MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -135,7 +135,6 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:status",
     ],
@@ -317,6 +316,7 @@ cc_library(
 cc_test(
     name = "image_cropping_calculator_test",
     srcs = ["image_cropping_calculator_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":image_cropping_calculator",
         ":image_cropping_calculator_cc_proto",
@@ -650,6 +650,7 @@ cc_library(
 cc_test(
     name = "segmentation_smoothing_calculator_test",
     srcs = ["segmentation_smoothing_calculator_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":image_clone_calculator",
         ":image_clone_calculator_cc_proto",
@@ -771,7 +772,10 @@ cc_test(
         "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
         "//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
     ],
-    tags = ["desktop_only_test"],
+    tags = [
+        "desktop_only_test",
+        "not_run:arm",
+    ],
     deps = [
         ":affine_transformation",
         ":image_transformation_calculator",

mediapipe/calculators/image/yuv_to_image_calculator.cc

@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
   buf[0] = (fourcc >> 24) & 0xff;
   buf[1] = (fourcc >> 16) & 0xff;
   buf[2] = (fourcc >> 8) & 0xff;
-  buf[3] = (fourcc)&0xff;
+  buf[3] = (fourcc) & 0xff;
   buf[4] = 0;
   return std::string(buf);
 }

mediapipe/calculators/tensor/BUILD

@@ -228,7 +228,6 @@ cc_library(
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
@@ -280,7 +279,6 @@ cc_library(
         "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
     ],
     alwayslink = 1,
 )
@@ -394,7 +392,7 @@ mediapipe_proto_library(
 # If you want to have precise control of which implementations to include (e.g. for strict binary
 # size concerns), depend on those implementations directly, and do not depend on
 # :inference_calculator.
-# In all cases, use "InferenceCalulator" in your graphs.
+# In all cases, use "InferenceCalculator" in your graphs.
 cc_library_with_tflite(
     name = "inference_calculator_interface",
     srcs = ["inference_calculator.cc"],
@@ -655,6 +653,11 @@ cc_library(
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": ["tensor_converter_calculator_gpu_deps"],
+    }) + select({
+        "//mediapipe:apple": [
+            "//third_party/apple_frameworks:MetalKit",
+        ],
+        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -1052,6 +1055,7 @@ cc_test(
         "testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
         "testdata/image_to_tensor/noop_except_range.png",
     ],
+    tags = ["not_run:arm"],
     deps = [
         ":image_to_tensor_calculator",
         ":image_to_tensor_converter",

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -282,18 +282,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
   if (options.has_volume_gain_db()) {
     gain_ = pow(10, options.volume_gain_db() / 20.0);
   }
-  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+  if (options.has_source_sample_rate()) {
-            !kAudioIn(cc).Header().IsEmpty())
+    source_sample_rate_ = options.source_sample_rate();
-      << "Must either specify the time series header of the \"AUDIO\" stream "
+  } else {
-         "or have the \"SAMPLE_RATE\" stream connected.";
+    RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
-  if (!kAudioIn(cc).Header().IsEmpty()) {
+              !kAudioIn(cc).Header().IsEmpty())
-    mediapipe::TimeSeriesHeader input_header;
+        << "Must either specify the time series header of the \"AUDIO\" stream "
-    MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+           "or have the \"SAMPLE_RATE\" stream connected.";
-        kAudioIn(cc).Header(), &input_header));
+    if (!kAudioIn(cc).Header().IsEmpty()) {
-    if (stream_mode_) {
+      mediapipe::TimeSeriesHeader input_header;
-      MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      MP_RETURN_IF_ERROR(
-    } else {
+          mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
-      source_sample_rate_ = input_header.sample_rate();
+              kAudioIn(cc).Header(), &input_header));
+      if (stream_mode_) {
+        MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      } else {
+        source_sample_rate_ = input_header.sample_rate();
+      }
     }
   }
   AppendZerosToSampleBuffer(padding_samples_before_);

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
   // The volume gain, measured in dB.
   // Scale the input audio amplitude by 10^(volume_gain_db/20).
   optional double volume_gain_db = 12;
+
+  // The source number of samples per second (hertz) of the input audio buffers.
+  optional double source_sample_rate = 13;
 }

mediapipe/calculators/tensor/bert_preprocessor_calculator.cc

@@ -22,7 +22,6 @@
 
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
@@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
   input_tensors.reserve(kNumInputTensorsForBert);
   for (int i = 0; i < kNumInputTensorsForBert; ++i) {
     input_tensors.push_back(
-        {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
+        {Tensor::ElementType::kInt32,
+         Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
   }
   std::memcpy(input_tensors[input_ids_tensor_index_]
                   .GetCpuWriteView()

mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc

@@ -96,6 +96,19 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
     CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
   // Read CPU input into tensors.
   RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
+
+  // If the input tensors have dynamic shape, then the tensors need to be
+  // resized and reallocated before we can copy the tensor values.
+  bool resized_tensor_shapes = false;
+  for (int i = 0; i < input_tensors.size(); ++i) {
+    if (input_tensors[i].shape().is_dynamic) {
+      interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims);
+      resized_tensor_shapes = true;
+    }
+  }
+  // Reallocation is needed for memory sanity.
+  if (resized_tensor_shapes) interpreter_->AllocateTensors();
+
   for (int i = 0; i < input_tensors.size(); ++i) {
     const TfLiteType input_tensor_type =
         interpreter_->tensor(interpreter_->inputs()[i])->type;

mediapipe/calculators/tensor/regex_preprocessor_calculator.cc

@@ -20,7 +20,6 @@
 #include <vector>
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
@@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
   // not found in the tokenizer vocab.
   std::vector<Tensor> result;
   result.push_back(
-      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
+      {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})});
   std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
               input_tokens.data(), input_tokens.size() * sizeof(int32_t));
   kTensorsOut(cc).Send(std::move(result));

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -256,6 +256,7 @@ class TensorsToDetectionsCalculator : public Node {
 
   bool gpu_inited_ = false;
   bool gpu_input_ = false;
+  bool gpu_has_enough_work_groups_ = true;
   bool anchors_init_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator);
@@ -291,7 +292,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
   auto output_detections = absl::make_unique<std::vector<Detection>>();
   bool gpu_processing = false;
-  if (CanUseGpu()) {
+  if (CanUseGpu() && gpu_has_enough_work_groups_) {
     // Use GPU processing only if at least one input tensor is already on GPU
     // (to avoid CPU->GPU overhead).
     for (const auto& tensor : *kInTensors(cc)) {
@@ -321,11 +322,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
     RET_CHECK(!has_custom_box_indices_);
   }
 
-  if (gpu_processing) {
+  if (gpu_processing && !gpu_inited_) {
-    if (!gpu_inited_) {
+    auto status = GpuInit(cc);
-      MP_RETURN_IF_ERROR(GpuInit(cc));
+    if (status.ok()) {
       gpu_inited_ = true;
+    } else if (status.code() == absl::StatusCode::kFailedPrecondition) {
+      // For initialization error because of hardware limitation, fallback to
+      // CPU processing.
+      LOG(WARNING) << status.message();
+    } else {
+      // For other error, let the error propagates.
+      return status;
     }
+  }
+  if (gpu_processing && gpu_inited_) {
     MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get()));
   } else {
     MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get()));
@@ -346,17 +356,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
     // TODO: Add flexible input tensor size handling.
     auto raw_box_tensor =
         &input_tensors[tensor_mapping_.detections_tensor_index()];
-    RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
     RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+    if (raw_box_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+      // The tensors from CPU inference has dim 3.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+    } else if (raw_box_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of box Tensor must be 3 or 4.");
+    }
     auto raw_score_tensor =
         &input_tensors[tensor_mapping_.scores_tensor_index()];
-    RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3);
+    if (raw_score_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      // The tensors from CPU inference has dim 3.
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+    } else if (raw_score_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of score Tensor must be 3 or 4.");
+    }
     auto raw_box_view = raw_box_tensor->GetCpuReadView();
     auto raw_boxes = raw_box_view.buffer<float>();
     auto raw_scores_view = raw_score_tensor->GetCpuReadView();
@@ -1111,8 +1145,13 @@ void main() {
     int max_wg_size;  //  typically <= 1024
     glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
                     &max_wg_size);  // y-dim
-    CHECK_LT(num_classes_, max_wg_size)
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
-        << "# classes must be < " << max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
     // TODO support better filtering.
     if (class_index_set_.is_allowlist) {
       CHECK_EQ(class_index_set_.values.size(),
@@ -1370,7 +1409,13 @@ kernel void scoreKernel(
         Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2});
     // # filter classes supported is hardware dependent.
     int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup;
-    CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
   }
 
 #endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)

mediapipe/calculators/tensorflow/BUILD

@@ -1077,6 +1077,7 @@ cc_test(
     linkstatic = 1,
     deps = [
         ":tensor_to_image_frame_calculator",
+        ":tensor_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:image_frame",

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc

@@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase {
 
  private:
   float scale_factor_;
+  bool scale_per_frame_min_max_;
 };
 
 REGISTER_CALCULATOR(TensorToImageFrameCalculator);
@@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
 absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) {
   scale_factor_ =
       cc->Options<TensorToImageFrameCalculatorOptions>().scale_factor();
+  scale_per_frame_min_max_ = cc->Options<TensorToImageFrameCalculatorOptions>()
+                                 .scale_per_frame_min_max();
   cc->SetOffset(TimestampDiff(0));
   return absl::OkStatus();
 }
@@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
   auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
   const int32_t total_size = height * width * depth;
 
+  if (scale_per_frame_min_max_) {
+    RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT)
+        << "Setting scale_per_frame_min_max requires FLOAT input tensors.";
+  }
   ::std::unique_ptr<const ImageFrame> output;
   if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
     // Allocate buffer with alignments.
     std::unique_ptr<uint8_t[]> buffer(
         new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
     auto data = input_tensor.flat<float>().data();
+    float min = 1e23;
+    float max = -1e23;
+    if (scale_per_frame_min_max_) {
+      for (int i = 0; i < total_size; ++i) {
+        float d = scale_factor_ * data[i];
+        if (d < min) {
+          min = d;
+        }
+        if (d > max) {
+          max = d;
+        }
+      }
+    }
     for (int i = 0; i < total_size; ++i) {
-      float d = scale_factor_ * data[i];
+      float d = data[i];
-      if (d < 0) d = 0;
+      if (scale_per_frame_min_max_) {
-      if (d > 255) d = 255;
+        d = 255 * (d - min) / (max - min + 1e-9);
+      } else {
+        d = scale_factor_ * d;
+        if (d < 0) d = 0;
+        if (d > 255) d = 255;
+      }
       buffer[i] = d;
     }
     output = ::absl::make_unique<ImageFrame>(

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto

@@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions {
   // Multiples floating point tensor outputs by this value before converting to
   // uint8. This is useful for converting from range [0, 1] to [0, 255]
   optional float scale_factor = 1 [default = 1.0];
+
+  // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255]
+  // per frame. This overrides any explicit scale_factor.
+  optional bool scale_per_frame_min_max = 2 [default = false];
 }

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc

@@ -11,7 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <type_traits>
 
+#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE";
 template <class TypeParam>
 class TensorToImageFrameCalculatorTest : public ::testing::Test {
  protected:
-  void SetUpRunner() {
+  void SetUpRunner(bool scale_per_frame_min_max = false) {
     CalculatorGraphConfig::Node config;
     config.set_calculator("TensorToImageFrameCalculator");
     config.add_input_stream("TENSOR:input_tensor");
     config.add_output_stream("IMAGE:output_image");
+    config.mutable_options()
+        ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext)
+        ->set_scale_per_frame_min_max(scale_per_frame_min_max);
     runner_ = absl::make_unique<CalculatorRunner>(config);
   }
 
@@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest,
   }
 }
 
+TYPED_TEST(TensorToImageFrameCalculatorTest,
+           Converts3DTensorToImageFrame2DGrayWithScaling) {
+  this->SetUpRunner(true);
+  auto& runner = this->runner_;
+  constexpr int kWidth = 16;
+  constexpr int kHeight = 8;
+  const tf::TensorShape tensor_shape{kHeight, kWidth};
+  auto tensor = absl::make_unique<tf::Tensor>(
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
+
+  // Writing sequence of integers as floats which we want normalized.
+  tensor_vec[0] = 255;
+  for (int i = 1; i < kWidth * kHeight; ++i) {
+    tensor_vec[i] = 200;
+  }
+
+  const int64_t time = 1234;
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  if (!std::is_same<TypeParam, float>::value) {
+    EXPECT_FALSE(runner->Run().ok());
+    return;  // Short circuit because does not apply to other types.
+  } else {
+    EXPECT_TRUE(runner->Run().ok());
+    const std::vector<Packet>& output_packets =
+        runner->Outputs().Tag(kImage).packets;
+    EXPECT_EQ(1, output_packets.size());
+    EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+    const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+    EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
+    EXPECT_EQ(kWidth, output_image.Width());
+    EXPECT_EQ(kHeight, output_image.Height());
+
+    EXPECT_EQ(255, output_image.PixelData()[0]);
+    for (int i = 1; i < kWidth * kHeight; ++i) {
+      const uint8_t pixel_value = output_image.PixelData()[i];
+      ASSERT_EQ(0, pixel_value);
+    }
+  }
+}
+
 }  // namespace mediapipe

mediapipe/calculators/util/BUILD

@@ -899,16 +899,77 @@ mediapipe_proto_library(
 cc_library(
     name = "landmarks_smoothing_calculator",
     srcs = ["landmarks_smoothing_calculator.cc"],
+    hdrs = ["landmarks_smoothing_calculator.h"],
     deps = [
         ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "landmarks_smoothing_calculator_utils",
+    srcs = ["landmarks_smoothing_calculator_utils.cc"],
+    hdrs = ["landmarks_smoothing_calculator_utils.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util/filtering:one_euro_filter",
         "//mediapipe/util/filtering:relative_velocity_filter",
-        "@com_google_absl//absl/algorithm:container",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "landmarks_smoothing_calculator_utils_test",
+    size = "small",
+    srcs = ["landmarks_smoothing_calculator_utils_test.cc"],
+    deps = [
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "multi_landmarks_smoothing_calculator",
+    srcs = ["multi_landmarks_smoothing_calculator.cc"],
+    hdrs = ["multi_landmarks_smoothing_calculator.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "multi_world_landmarks_smoothing_calculator",
+    srcs = ["multi_world_landmarks_smoothing_calculator.cc"],
+    hdrs = ["multi_world_landmarks_smoothing_calculator.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2020 The MediaPipe Authors.
+// Copyright 2023 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,471 +12,105 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.h"
+
 #include <memory>
 
-#include "absl/algorithm/container.h"
 #include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
-#include "mediapipe/util/filtering/one_euro_filter.h"
-#include "mediapipe/util/filtering/relative_velocity_filter.h"
 
 namespace mediapipe {
+namespace api2 {
 
 namespace {
 
-constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
-constexpr char kLandmarksTag[] = "LANDMARKS";
-constexpr char kImageSizeTag[] = "IMAGE_SIZE";
-constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI";
-constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
-constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
-
 using ::mediapipe::NormalizedRect;
-using mediapipe::OneEuroFilter;
 using ::mediapipe::Rect;
-using mediapipe::RelativeVelocityFilter;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
-
+using ::mediapipe::landmarks_smoothing::InitializeLandmarksFilter;
-void NormalizedLandmarksToLandmarks(
+using ::mediapipe::landmarks_smoothing::LandmarksFilter;
-    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
-    const int image_height, LandmarkList* landmarks) {
+using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
-  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
-    const auto& norm_landmark = norm_landmarks.landmark(i);
-
-    auto* landmark = landmarks->add_landmark();
-    landmark->set_x(norm_landmark.x() * image_width);
-    landmark->set_y(norm_landmark.y() * image_height);
-    // Scale Z the same way as X (using image width).
-    landmark->set_z(norm_landmark.z() * image_width);
-    landmark->set_visibility(norm_landmark.visibility());
-    landmark->set_presence(norm_landmark.presence());
-  }
-}
-
-void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
-                                    const int image_width,
-                                    const int image_height,
-                                    NormalizedLandmarkList* norm_landmarks) {
-  for (int i = 0; i < landmarks.landmark_size(); ++i) {
-    const auto& landmark = landmarks.landmark(i);
-
-    auto* norm_landmark = norm_landmarks->add_landmark();
-    norm_landmark->set_x(landmark.x() / image_width);
-    norm_landmark->set_y(landmark.y() / image_height);
-    // Scale Z the same way as X (using image width).
-    norm_landmark->set_z(landmark.z() / image_width);
-    norm_landmark->set_visibility(landmark.visibility());
-    norm_landmark->set_presence(landmark.presence());
-  }
-}
-
-// Estimate object scale to use its inverse value as velocity scale for
-// RelativeVelocityFilter. If value will be too small (less than
-// `options_.min_allowed_object_scale`) smoothing will be disabled and
-// landmarks will be returned as is.
-// Object scale is calculated as average between bounding box width and height
-// with sides parallel to axis.
-float GetObjectScale(const LandmarkList& landmarks) {
-  const auto& lm_minmax_x = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.x() < b.x(); });
-  const float x_min = lm_minmax_x.first->x();
-  const float x_max = lm_minmax_x.second->x();
-
-  const auto& lm_minmax_y = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.y() < b.y(); });
-  const float y_min = lm_minmax_y.first->y();
-  const float y_max = lm_minmax_y.second->y();
-
-  const float object_width = x_max - x_min;
-  const float object_height = y_max - y_min;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const NormalizedRect& roi, const int image_width,
-                     const int image_height) {
-  const float object_width = roi.width() * image_width;
-  const float object_height = roi.height() * image_height;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const Rect& roi) {
-  return (roi.width() + roi.height()) / 2.0f;
-}
-
-// Abstract class for various landmarks filters.
-class LandmarksFilter {
- public:
-  virtual ~LandmarksFilter() = default;
-
-  virtual absl::Status Reset() { return absl::OkStatus(); }
-
-  virtual absl::Status Apply(const LandmarkList& in_landmarks,
-                             const absl::Duration& timestamp,
-                             const absl::optional<float> object_scale_opt,
-                             LandmarkList* out_landmarks) = 0;
-};
-
-// Returns landmarks as is without smoothing.
-class NoFilter : public LandmarksFilter {
- public:
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    *out_landmarks = in_landmarks;
-    return absl::OkStatus();
-  }
-};
-
-// Please check RelativeVelocityFilter documentation for details.
-class VelocityFilter : public LandmarksFilter {
- public:
-  VelocityFilter(int window_size, float velocity_scale,
-                 float min_allowed_object_scale, bool disable_value_scaling)
-      : window_size_(window_size),
-        velocity_scale_(velocity_scale),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    x_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    y_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    z_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-
-    return absl::OkStatus();
-  }
-
-  int window_size_;
-  float velocity_scale_;
-  float min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<RelativeVelocityFilter> x_filters_;
-  std::vector<RelativeVelocityFilter> y_filters_;
-  std::vector<RelativeVelocityFilter> z_filters_;
-};
-
-// Please check OneEuroFilter documentation for details.
-class OneEuroFilterImpl : public LandmarksFilter {
- public:
-  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
-                    double derivate_cutoff, float min_allowed_object_scale,
-                    bool disable_value_scaling)
-      : frequency_(frequency),
-        min_cutoff_(min_cutoff),
-        beta_(beta),
-        derivate_cutoff_(derivate_cutoff),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    for (int i = 0; i < n_landmarks; ++i) {
-      x_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      y_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      z_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-    }
-
-    return absl::OkStatus();
-  }
-
-  double frequency_;
-  double min_cutoff_;
-  double beta_;
-  double derivate_cutoff_;
-  double min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<OneEuroFilter> x_filters_;
-  std::vector<OneEuroFilter> y_filters_;
-  std::vector<OneEuroFilter> z_filters_;
-};
 
 }  // namespace
 
-// A calculator to smooth landmarks over time.
+class LandmarksSmoothingCalculatorImpl
-//
+    : public NodeImpl<LandmarksSmoothingCalculator> {
-// Inputs:
-//   NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
-//   IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
-//     Required to perform all computations in absolute coordinates to avoid any
-//     influence of normalized values.
-//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
-//     input landmarks) used to determine the object scale for some of the
-//     filters. If not provided - object scale will be calculated from
-//     landmarks.
-//
-// Outputs:
-//   NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
-//
-// Example config:
-//   node {
-//     calculator: "LandmarksSmoothingCalculator"
-//     input_stream: "NORM_LANDMARKS:pose_landmarks"
-//     input_stream: "IMAGE_SIZE:image_size"
-//     input_stream: "OBJECT_SCALE_ROI:roi"
-//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
-//     options: {
-//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
-//         velocity_filter: {
-//           window_size: 5
-//           velocity_scale: 10.0
-//         }
-//       }
-//     }
-//   }
-//
-class LandmarksSmoothingCalculator : public CalculatorBase {
  public:
-  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override {
-  absl::Status Open(CalculatorContext* cc) override;
+    ASSIGN_OR_RETURN(landmarks_filter_,
-  absl::Status Process(CalculatorContext* cc) override;
+                     InitializeLandmarksFilter(
+                         cc->Options<LandmarksSmoothingCalculatorOptions>()));
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if ((kInNormLandmarks(cc).IsConnected() &&
+         kInNormLandmarks(cc).IsEmpty()) ||
+        (kInLandmarks(cc).IsConnected() && kInLandmarks(cc).IsEmpty())) {
+      MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    if (kInNormLandmarks(cc).IsConnected()) {
+      const auto& in_norm_landmarks = kInNormLandmarks(cc).Get();
+
+      int image_width;
+      int image_height;
+      std::tie(image_width, image_height) = kImageSize(cc).Get();
+
+      absl::optional<float> object_scale;
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<NormalizedRect>();
+        object_scale = GetObjectScale(roi, image_width, image_height);
+      }
+
+      auto in_landmarks = absl::make_unique<LandmarkList>();
+      NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width,
+                                     image_height, *in_landmarks.get());
+
+      auto out_landmarks = absl::make_unique<LandmarkList>();
+      MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
+          *in_landmarks, timestamp, object_scale, *out_landmarks));
+
+      auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
+      LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
+                                     *out_norm_landmarks.get());
+
+      kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks));
+    } else {
+      const auto& in_landmarks = kInLandmarks(cc).Get();
+
+      absl::optional<float> object_scale;
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<Rect>();
+        object_scale = GetObjectScale(roi);
+      }
+
+      auto out_landmarks = absl::make_unique<LandmarkList>();
+      MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
+          in_landmarks, timestamp, object_scale, *out_landmarks));
+
+      kOutLandmarks(cc).Send(std::move(out_landmarks));
+    }
+
+    return absl::OkStatus();
+  }
 
  private:
   std::unique_ptr<LandmarksFilter> landmarks_filter_;
 };
-REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksSmoothingCalculatorImpl);
-
-absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-    cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
-    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
-    cc->Outputs()
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Set<NormalizedLandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<NormalizedRect>();
-    }
-  } else {
-    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
-    cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<Rect>();
-    }
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
-  cc->SetOffset(TimestampDiff(0));
-
-  // Pick landmarks filter.
-  const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
-  if (options.has_no_filter()) {
-    landmarks_filter_ = absl::make_unique<NoFilter>();
-  } else if (options.has_velocity_filter()) {
-    landmarks_filter_ = absl::make_unique<VelocityFilter>(
-        options.velocity_filter().window_size(),
-        options.velocity_filter().velocity_scale(),
-        options.velocity_filter().min_allowed_object_scale(),
-        options.velocity_filter().disable_value_scaling());
-  } else if (options.has_one_euro_filter()) {
-    landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
-        options.one_euro_filter().frequency(),
-        options.one_euro_filter().min_cutoff(),
-        options.one_euro_filter().beta(),
-        options.one_euro_filter().derivate_cutoff(),
-        options.one_euro_filter().min_allowed_object_scale(),
-        options.one_euro_filter().disable_value_scaling());
-  } else {
-    RET_CHECK_FAIL()
-        << "Landmarks filter is either not specified or not supported";
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
-  // Check that landmarks are not empty and reset the filter if so.
-  // Don't emit an empty packet for this timestamp.
-  if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
-       cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
-      (cc->Inputs().HasTag(kLandmarksTag) &&
-       cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
-    MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
-    return absl::OkStatus();
-  }
-
-  const auto& timestamp =
-      absl::Microseconds(cc->InputTimestamp().Microseconds());
-
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-    const auto& in_norm_landmarks =
-        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
-
-    int image_width;
-    int image_height;
-    std::tie(image_width, image_height) =
-        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
-
-    absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<NormalizedRect>();
-      object_scale = GetObjectScale(roi, image_width, image_height);
-    }
-
-    auto in_landmarks = absl::make_unique<LandmarkList>();
-    NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
-                                   in_landmarks.get());
-
-    auto out_landmarks = absl::make_unique<LandmarkList>();
-    MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        *in_landmarks, timestamp, object_scale, out_landmarks.get()));
-
-    auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
-    LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
-                                   out_norm_landmarks.get());
-
-    cc->Outputs()
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Add(out_norm_landmarks.release(), cc->InputTimestamp());
-  } else {
-    const auto& in_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
-
-    absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<Rect>();
-      object_scale = GetObjectScale(roi);
-    }
-
-    auto out_landmarks = absl::make_unique<LandmarkList>();
-    MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        in_landmarks, timestamp, object_scale, out_landmarks.get()));
-
-    cc->Outputs()
-        .Tag(kFilteredLandmarksTag)
-        .Add(out_landmarks.release(), cc->InputTimestamp());
-  }
-
-  return absl::OkStatus();
-}
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
+//     to smooth.
+//   LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
+//   IMAGE_SIZE (optional): A std::pair<int, int> represention of image width
+//     and height. Required to perform all computations in absolute coordinates
+//     when smoothing NORM_LANDMARKS to avoid any influence of normalized
+//     values.
+//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
+//     input landmarks) used to determine the object scale for some of the
+//     filters. If not provided - object scale will be calculated from
+//     landmarks.
+//
+// Outputs:
+//   NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
+//     landmarks.
+//   FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "LandmarksSmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:landmarks"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "NORM_FILTERED_LANDMARKS:landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class LandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
+      kInNormLandmarks{"NORM_LANDMARKS"};
+  static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarks{
+      "LANDMARKS"};
+  static constexpr Input<std::pair<int, int>>::Optional kImageSize{
+      "IMAGE_SIZE"};
+  static constexpr Input<OneOf<NormalizedRect, Rect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<mediapipe::NormalizedLandmarkList>::Optional
+      kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
+  static constexpr Output<mediapipe::LandmarkList>::Optional kOutLandmarks{
+      "FILTERED_LANDMARKS"};
+  MEDIAPIPE_NODE_INTERFACE(LandmarksSmoothingCalculator, kInNormLandmarks,
+                           kInLandmarks, kImageSize, kObjectScaleRoi,
+                           kOutNormLandmarks, kOutLandmarks);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK(kInNormLandmarks(cc).IsConnected() ^
+              kInLandmarks(cc).IsConnected())
+        << "One and only one of NORM_LANDMARKS and LANDMARKS input is allowed";
+
+    // TODO: Verify scale ROI is of the same type as landmarks
+    // that are being smoothed.
+
+    if (kInNormLandmarks(cc).IsConnected()) {
+      RET_CHECK(kImageSize(cc).IsConnected());
+      RET_CHECK(kOutNormLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutLandmarks(cc).IsConnected());
+    } else {
+      RET_CHECK(!kImageSize(cc).IsConnected());
+      RET_CHECK(kOutLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutNormLandmarks(cc).IsConnected());
+    }
+
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc

@@ -0,0 +1,375 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+
+#include <iostream>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+namespace {
+
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::OneEuroFilter;
+using ::mediapipe::Rect;
+using ::mediapipe::RelativeVelocityFilter;
+
+// Estimate object scale to use its inverse value as velocity scale for
+// RelativeVelocityFilter. If value will be too small (less than
+// `options_.min_allowed_object_scale`) smoothing will be disabled and
+// landmarks will be returned as is.
+// Object scale is calculated as average between bounding box width and height
+// with sides parallel to axis.
+float GetObjectScale(const LandmarkList& landmarks) {
+  const auto& lm_minmax_x = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.x() < b.x(); });
+  const float x_min = lm_minmax_x.first->x();
+  const float x_max = lm_minmax_x.second->x();
+
+  const auto& lm_minmax_y = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.y() < b.y(); });
+  const float y_min = lm_minmax_y.first->y();
+  const float y_max = lm_minmax_y.second->y();
+
+  const float object_width = x_max - x_min;
+  const float object_height = y_max - y_min;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+// Returns landmarks as is without smoothing.
+class NoFilter : public LandmarksFilter {
+ public:
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    out_landmarks = in_landmarks;
+    return absl::OkStatus();
+  }
+};
+
+// Please check RelativeVelocityFilter documentation for details.
+class VelocityFilter : public LandmarksFilter {
+ public:
+  VelocityFilter(int window_size, float velocity_scale,
+                 float min_allowed_object_scale, bool disable_value_scaling)
+      : window_size_(window_size),
+        velocity_scale_(velocity_scale),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    x_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    y_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    z_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+
+    return absl::OkStatus();
+  }
+
+  int window_size_;
+  float velocity_scale_;
+  float min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<RelativeVelocityFilter> x_filters_;
+  std::vector<RelativeVelocityFilter> y_filters_;
+  std::vector<RelativeVelocityFilter> z_filters_;
+};
+
+// Please check OneEuroFilter documentation for details.
+class OneEuroFilterImpl : public LandmarksFilter {
+ public:
+  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
+                    double derivate_cutoff, float min_allowed_object_scale,
+                    bool disable_value_scaling)
+      : frequency_(frequency),
+        min_cutoff_(min_cutoff),
+        beta_(beta),
+        derivate_cutoff_(derivate_cutoff),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    for (int i = 0; i < n_landmarks; ++i) {
+      x_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      y_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      z_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+    }
+
+    return absl::OkStatus();
+  }
+
+  double frequency_;
+  double min_cutoff_;
+  double beta_;
+  double derivate_cutoff_;
+  double min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<OneEuroFilter> x_filters_;
+  std::vector<OneEuroFilter> y_filters_;
+  std::vector<OneEuroFilter> z_filters_;
+};
+
+}  // namespace
+
+void NormalizedLandmarksToLandmarks(
+    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+    const int image_height, LandmarkList& landmarks) {
+  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
+    const auto& norm_landmark = norm_landmarks.landmark(i);
+
+    auto* landmark = landmarks.add_landmark();
+    landmark->set_x(norm_landmark.x() * image_width);
+    landmark->set_y(norm_landmark.y() * image_height);
+    // Scale Z the same way as X (using image width).
+    landmark->set_z(norm_landmark.z() * image_width);
+
+    if (norm_landmark.has_visibility()) {
+      landmark->set_visibility(norm_landmark.visibility());
+    } else {
+      landmark->clear_visibility();
+    }
+
+    if (norm_landmark.has_presence()) {
+      landmark->set_presence(norm_landmark.presence());
+    } else {
+      landmark->clear_presence();
+    }
+  }
+}
+
+void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
+                                    const int image_width,
+                                    const int image_height,
+                                    NormalizedLandmarkList& norm_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+
+    auto* norm_landmark = norm_landmarks.add_landmark();
+    norm_landmark->set_x(landmark.x() / image_width);
+    norm_landmark->set_y(landmark.y() / image_height);
+    // Scale Z the same way as X (using image width).
+    norm_landmark->set_z(landmark.z() / image_width);
+
+    if (landmark.has_visibility()) {
+      norm_landmark->set_visibility(landmark.visibility());
+    } else {
+      norm_landmark->clear_visibility();
+    }
+
+    if (landmark.has_presence()) {
+      norm_landmark->set_presence(landmark.presence());
+    } else {
+      norm_landmark->clear_presence();
+    }
+  }
+}
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height) {
+  const float object_width = roi.width() * image_width;
+  const float object_height = roi.height() * image_height;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+float GetObjectScale(const Rect& roi) {
+  return (roi.width() + roi.height()) / 2.0f;
+}
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const LandmarksSmoothingCalculatorOptions& options) {
+  if (options.has_no_filter()) {
+    return absl::make_unique<NoFilter>();
+  } else if (options.has_velocity_filter()) {
+    return absl::make_unique<VelocityFilter>(
+        options.velocity_filter().window_size(),
+        options.velocity_filter().velocity_scale(),
+        options.velocity_filter().min_allowed_object_scale(),
+        options.velocity_filter().disable_value_scaling());
+  } else if (options.has_one_euro_filter()) {
+    return absl::make_unique<OneEuroFilterImpl>(
+        options.one_euro_filter().frequency(),
+        options.one_euro_filter().min_cutoff(),
+        options.one_euro_filter().beta(),
+        options.one_euro_filter().derivate_cutoff(),
+        options.one_euro_filter().min_allowed_object_scale(),
+        options.one_euro_filter().disable_value_scaling());
+  } else {
+    RET_CHECK_FAIL()
+        << "Landmarks filter is either not specified or not supported";
+  }
+}
+
+absl::StatusOr<LandmarksFilter*> MultiLandmarkFilters::GetOrCreate(
+    const int64_t tracking_id,
+    const mediapipe::LandmarksSmoothingCalculatorOptions& options) {
+  const auto it = filters_.find(tracking_id);
+  if (it != filters_.end()) {
+    return it->second.get();
+  }
+
+  ASSIGN_OR_RETURN(auto landmarks_filter, InitializeLandmarksFilter(options));
+  filters_[tracking_id] = std::move(landmarks_filter);
+  return filters_[tracking_id].get();
+}
+
+void MultiLandmarkFilters::ClearUnused(
+    const std::vector<int64_t>& tracking_ids) {
+  std::vector<int64_t> unused_tracking_ids;
+  for (const auto& it : filters_) {
+    bool unused = true;
+    for (int64_t tracking_id : tracking_ids) {
+      if (tracking_id == it.first) unused = false;
+    }
+    if (unused) unused_tracking_ids.push_back(it.first);
+  }
+
+  for (int64_t tracking_id : unused_tracking_ids) {
+    filters_.erase(tracking_id);
+  }
+}
+
+void MultiLandmarkFilters::Clear() { filters_.clear(); }
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h

@@ -0,0 +1,77 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+void NormalizedLandmarksToLandmarks(
+    const mediapipe::NormalizedLandmarkList& norm_landmarks,
+    const int image_width, const int image_height,
+    mediapipe::LandmarkList& landmarks);
+
+void LandmarksToNormalizedLandmarks(
+    const mediapipe::LandmarkList& landmarks, const int image_width,
+    const int image_height, mediapipe::NormalizedLandmarkList& norm_landmarks);
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height);
+
+float GetObjectScale(const Rect& roi);
+
+// Abstract class for various landmarks filters.
+class LandmarksFilter {
+ public:
+  virtual ~LandmarksFilter() = default;
+
+  virtual absl::Status Reset() { return absl::OkStatus(); }
+
+  virtual absl::Status Apply(const mediapipe::LandmarkList& in_landmarks,
+                             const absl::Duration& timestamp,
+                             const absl::optional<float> object_scale_opt,
+                             mediapipe::LandmarkList& out_landmarks) = 0;
+};
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const mediapipe::LandmarksSmoothingCalculatorOptions& options);
+
+class MultiLandmarkFilters {
+ public:
+  virtual ~MultiLandmarkFilters() = default;
+
+  virtual absl::StatusOr<LandmarksFilter*> GetOrCreate(
+      const int64_t tracking_id,
+      const mediapipe::LandmarksSmoothingCalculatorOptions& options);
+
+  virtual void ClearUnused(const std::vector<int64_t>& tracking_ids);
+
+  virtual void Clear();
+
+ private:
+  std::map<int64_t, std::unique_ptr<LandmarksFilter>> filters_;
+};
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_

mediapipe/calculators/util/landmarks_smoothing_calculator_utils_test.cc

@@ -0,0 +1,118 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+namespace {
+
+TEST(LandmarksSmoothingCalculatorUtilsTest, NormalizedLandmarksToLandmarks) {
+  NormalizedLandmarkList norm_landmarks;
+  NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
+  norm_landmark->set_x(0.1);
+  norm_landmark->set_y(0.2);
+  norm_landmark->set_z(0.3);
+  norm_landmark->set_visibility(0.4);
+  norm_landmark->set_presence(0.5);
+
+  LandmarkList landmarks;
+  NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, landmarks);
+
+  EXPECT_EQ(landmarks.landmark_size(), 1);
+  Landmark landmark = landmarks.landmark(0);
+  EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
+  EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
+  EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
+  EXPECT_NEAR(landmark.visibility(), 0.4, 1e-6);
+  EXPECT_NEAR(landmark.presence(), 0.5, 1e-6);
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest,
+     NormalizedLandmarksToLandmarks_EmptyVisibilityAndPresence) {
+  NormalizedLandmarkList norm_landmarks;
+  NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
+  norm_landmark->set_x(0.1);
+  norm_landmark->set_y(0.2);
+  norm_landmark->set_z(0.3);
+  norm_landmark->clear_visibility();
+  norm_landmark->clear_presence();
+
+  LandmarkList landmarks;
+  NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, landmarks);
+
+  EXPECT_EQ(landmarks.landmark_size(), 1);
+  Landmark landmark = landmarks.landmark(0);
+  EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
+  EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
+  EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
+  EXPECT_FALSE(landmark.has_visibility());
+  EXPECT_FALSE(landmark.has_presence());
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest, LandmarksToNormalizedLandmarks) {
+  LandmarkList landmarks;
+  Landmark* landmark = landmarks.add_landmark();
+  landmark->set_x(1.0);
+  landmark->set_y(2.0);
+  landmark->set_z(3.0);
+  landmark->set_visibility(0.4);
+  landmark->set_presence(0.5);
+
+  NormalizedLandmarkList norm_landmarks;
+  LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, norm_landmarks);
+
+  EXPECT_EQ(norm_landmarks.landmark_size(), 1);
+  NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
+  EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
+  EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
+  EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
+  EXPECT_NEAR(norm_landmark.visibility(), 0.4, 1e-6);
+  EXPECT_NEAR(norm_landmark.presence(), 0.5, 1e-6);
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest,
+     LandmarksToNormalizedLandmarks_EmptyVisibilityAndPresence) {
+  LandmarkList landmarks;
+  Landmark* landmark = landmarks.add_landmark();
+  landmark->set_x(1.0);
+  landmark->set_y(2.0);
+  landmark->set_z(3.0);
+  landmark->clear_visibility();
+  landmark->clear_presence();
+
+  NormalizedLandmarkList norm_landmarks;
+  LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, norm_landmarks);
+
+  EXPECT_EQ(norm_landmarks.landmark_size(), 1);
+  NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
+  EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
+  EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
+  EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
+  EXPECT_FALSE(norm_landmark.has_visibility());
+  EXPECT_FALSE(norm_landmark.has_presence());
+}
+
+}  // namespace
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe

mediapipe/calculators/util/multi_landmarks_smoothing_calculator.cc

@@ -0,0 +1,113 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
+using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
+using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
+using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
+
+}  // namespace
+
+class MultiLandmarksSmoothingCalculatorImpl
+    : public NodeImpl<MultiLandmarksSmoothingCalculator> {
+ public:
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if (kInNormLandmarks(cc).IsEmpty()) {
+      multi_filters_.Clear();
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    const auto& tracking_ids = kTrackingIds(cc).Get();
+    multi_filters_.ClearUnused(tracking_ids);
+
+    const auto& in_norm_landmarks_vec = kInNormLandmarks(cc).Get();
+    RET_CHECK_EQ(in_norm_landmarks_vec.size(), tracking_ids.size());
+
+    int image_width;
+    int image_height;
+    std::tie(image_width, image_height) = kImageSize(cc).Get();
+
+    std::optional<std::vector<NormalizedRect>> object_scale_roi_vec;
+    if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+      object_scale_roi_vec = kObjectScaleRoi(cc).Get();
+      RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
+    }
+
+    std::vector<NormalizedLandmarkList> out_norm_landmarks_vec;
+    for (int i = 0; i < tracking_ids.size(); ++i) {
+      LandmarkList in_landmarks;
+      NormalizedLandmarksToLandmarks(in_norm_landmarks_vec[i], image_width,
+                                     image_height, in_landmarks);
+
+      std::optional<float> object_scale;
+      if (object_scale_roi_vec) {
+        object_scale = GetObjectScale(object_scale_roi_vec.value()[i],
+                                      image_width, image_height);
+      }
+
+      ASSIGN_OR_RETURN(auto* landmarks_filter,
+                       multi_filters_.GetOrCreate(
+                           tracking_ids[i],
+                           cc->Options<LandmarksSmoothingCalculatorOptions>()));
+
+      LandmarkList out_landmarks;
+      MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
+                                                 object_scale, out_landmarks));
+
+      NormalizedLandmarkList out_norm_landmarks;
+      LandmarksToNormalizedLandmarks(out_landmarks, image_width, image_height,
+                                     out_norm_landmarks);
+
+      out_norm_landmarks_vec.push_back(std::move(out_norm_landmarks));
+    }
+
+    kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks_vec));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  MultiLandmarkFilters multi_filters_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(MultiLandmarksSmoothingCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h

@@ -0,0 +1,81 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   NORM_LANDMARKS: A std::vector<NormalizedLandmarkList> of landmarks you want
+//     to smooth.
+//   TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
+//     landmarks over time. When new ID arrives - calculator will initialize new
+//     filter. When tracking ID is no longer provided - calculator will forget
+//     smoothing state.
+//   IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
+//     Required to perform all computations in absolute coordinates to avoid any
+//     influence of normalized values.
+//   OBJECT_SCALE_ROI (optional): A std::vector<NormRect> used to determine the
+//     object scale for some of the filters. If not provided - object scale will
+//     be calculated from landmarks.
+//
+// Outputs:
+//   NORM_FILTERED_LANDMARKS: A std::vector<NormalizedLandmarkList> of smoothed
+//     landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "MultiLandmarksSmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:pose_landmarks"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class MultiLandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<std::vector<mediapipe::NormalizedLandmarkList>>
+      kInNormLandmarks{"NORM_LANDMARKS"};
+  static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
+  static constexpr Input<std::pair<int, int>> kImageSize{"IMAGE_SIZE"};
+  static constexpr Input<std::vector<NormalizedRect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<std::vector<mediapipe::NormalizedLandmarkList>>
+      kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(MultiLandmarksSmoothingCalculator, kInNormLandmarks,
+                           kTrackingIds, kImageSize, kObjectScaleRoi,
+                           kOutNormLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.cc

@@ -0,0 +1,100 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+using ::mediapipe::Rect;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
+using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
+
+}  // namespace
+
+class MultiWorldLandmarksSmoothingCalculatorImpl
+    : public NodeImpl<MultiWorldLandmarksSmoothingCalculator> {
+ public:
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if (kInLandmarks(cc).IsEmpty()) {
+      multi_filters_.Clear();
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    const auto& tracking_ids = kTrackingIds(cc).Get();
+    multi_filters_.ClearUnused(tracking_ids);
+
+    const auto& in_landmarks_vec = kInLandmarks(cc).Get();
+    RET_CHECK_EQ(in_landmarks_vec.size(), tracking_ids.size());
+
+    std::optional<std::vector<Rect>> object_scale_roi_vec;
+    if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+      object_scale_roi_vec = kObjectScaleRoi(cc).Get();
+      RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
+    }
+
+    std::vector<LandmarkList> out_landmarks_vec;
+    for (int i = 0; i < tracking_ids.size(); ++i) {
+      const auto& in_landmarks = in_landmarks_vec[i];
+
+      std::optional<float> object_scale;
+      if (object_scale_roi_vec) {
+        object_scale = GetObjectScale(object_scale_roi_vec.value()[i]);
+      }
+
+      ASSIGN_OR_RETURN(auto* landmarks_filter,
+                       multi_filters_.GetOrCreate(
+                           tracking_ids[i],
+                           cc->Options<LandmarksSmoothingCalculatorOptions>()));
+
+      LandmarkList out_landmarks;
+      MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
+                                                 object_scale, out_landmarks));
+
+      out_landmarks_vec.push_back(std::move(out_landmarks));
+    }
+
+    kOutLandmarks(cc).Send(std::move(out_landmarks_vec));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  MultiLandmarkFilters multi_filters_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(MultiWorldLandmarksSmoothingCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h

@@ -0,0 +1,74 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   LANDMARKS: A std::vector<LandmarkList> of landmarks you want to
+//     smooth.
+//   TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
+//     landmarks over time. When new ID arrives - calculator will initialize new
+//     filter. When tracking ID is no longer provided - calculator will forget
+//     smoothing state.
+//   OBJECT_SCALE_ROI (optional): A std::vector<Rect> used to determine the
+//     object scale for some of the filters. If not provided - object scale will
+//     be calculated from landmarks.
+//
+// Outputs:
+//   FILTERED_LANDMARKS: A std::vector<LandmarkList> of smoothed landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "MultiWorldLandmarksSmoothingCalculator"
+//     input_stream: "LANDMARKS:landmarks"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "FILTERED_LANDMARKS:landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class MultiWorldLandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<std::vector<mediapipe::LandmarkList>> kInLandmarks{
+      "LANDMARKS"};
+  static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
+  static constexpr Input<std::vector<Rect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<std::vector<mediapipe::LandmarkList>> kOutLandmarks{
+      "FILTERED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(MultiWorldLandmarksSmoothingCalculator, kInLandmarks,
+                           kTrackingIds, kObjectScaleRoi, kOutLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc

@@ -124,7 +124,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     int center_row = out_lms.landmark(lm_index).y() * hm_height;
     // Point is outside of the image let's keep it intact.
     if (center_col < 0 || center_col >= hm_width || center_row < 0 ||
-        center_col >= hm_height) {
+        center_row >= hm_height) {
       continue;
     }
 

mediapipe/examples/ios/facedetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectioncpu",

mediapipe/examples/ios/facedetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectiongpu",

mediapipe/examples/ios/faceeffect/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "faceeffect",

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facemeshgpu",

mediapipe/examples/ios/handdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handdetectiongpu",

mediapipe/examples/ios/handtrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handtrackinggpu",

mediapipe/examples/ios/helloworld/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "helloworld",

mediapipe/examples/ios/holistictrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "holistictrackinggpu",

mediapipe/examples/ios/iristrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "iristrackinggpu",

mediapipe/examples/ios/objectdetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectioncpu",

mediapipe/examples/ios/objectdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiongpu",

mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiontrackinggpu",

mediapipe/examples/ios/posetrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "posetrackinggpu",

mediapipe/examples/ios/selfiesegmentationgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "selfiesegmentationgpu",

mediapipe/framework/BUILD

@@ -1099,6 +1099,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ],
+    alwayslink = True,  # Defines TestServiceCalculator
 )
 
 cc_library(
@@ -1354,6 +1355,23 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "calculator_graph_summary_packet_test",
+    srcs = ["calculator_graph_summary_packet_test.cc"],
+    deps = [
+        ":calculator_framework",
+        ":packet",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_test(
     name = "calculator_runner_test",
     size = "medium",
@@ -1405,6 +1423,7 @@ cc_test(
         "calculator_graph_test.cc",
     ],
     linkstatic = 1,
+    tags = ["not_run:arm"],
     visibility = ["//visibility:public"],
     deps = [
         ":calculator_framework",

mediapipe/framework/api2/builder.h

@@ -32,7 +32,7 @@ template <class T>
 struct dependent_false : std::false_type {};
 
 template <typename T>
-T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, int index) {
+T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, size_t index) {
   auto& vec = *vecp;
   if (vec.size() <= index) {
     vec.resize(index + 1);

mediapipe/framework/api2/node.h

@@ -88,8 +88,7 @@ struct NodeRegistrationStatic {
   static mediapipe::RegistrationToken Make() {
     return mediapipe::CalculatorBaseRegistry::Register(
         T::kCalculatorName,
-        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>,
+        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>);
-        __FILE__, __LINE__);
   }
 
   using RequireStatics = ForceStaticInstantiation<&registration>;
@@ -105,8 +104,8 @@ struct SubgraphRegistrationImpl {
   static NoDestructor<mediapipe::RegistrationToken> registration;
 
   static mediapipe::RegistrationToken Make() {
-    return mediapipe::SubgraphRegistry::Register(
+    return mediapipe::SubgraphRegistry::Register(T::kCalculatorName,
-        T::kCalculatorName, absl::make_unique<T>, __FILE__, __LINE__);
+                                                 absl::make_unique<T>);
   }
 
   using RequireStatics = ForceStaticInstantiation<&registration>;
@@ -224,13 +223,12 @@ class SubgraphImpl : public Subgraph, public Intf {
 
 // This macro is used to register a calculator that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                   \
+#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                  \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>                \
+  static mediapipe::NoDestructor<mediapipe::RegistrationToken>               \
-  REGISTRY_STATIC_VAR(calculator_registration,                                \
+  REGISTRY_STATIC_VAR(calculator_registration,                               \
-                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register(  \
+                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register( \
-      Impl::kCalculatorName,                                                  \
+      Impl::kCalculatorName,                                                 \
-      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>, \
+      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>))
-      __FILE__, __LINE__))
 
 // This macro is used to register a non-split-contract calculator. Deprecated.
 #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
@@ -241,7 +239,7 @@ class SubgraphImpl : public Subgraph, public Intf {
   static mediapipe::NoDestructor<mediapipe::RegistrationToken>         \
   REGISTRY_STATIC_VAR(subgraph_registration,                           \
                       __LINE__)(mediapipe::SubgraphRegistry::Register( \
-      Impl::kCalculatorName, absl::make_unique<Impl>, __FILE__, __LINE__))
+      Impl::kCalculatorName, absl::make_unique<Impl>))
 
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/api2/packet.h

@@ -165,7 +165,7 @@ template <class V, class... U>
 struct IsCompatibleType<V, OneOf<U...>>
     : std::integral_constant<bool, (std::is_same_v<V, U> || ...)> {};
 
-};  // namespace internal
+}  // namespace internal
 
 template <typename T>
 inline Packet<T> PacketBase::As() const {
@@ -259,19 +259,19 @@ struct First {
 
 template <class T>
 struct AddStatus {
-  using type = StatusOr<T>;
+  using type = absl::StatusOr<T>;
 };
 template <class T>
-struct AddStatus<StatusOr<T>> {
+struct AddStatus<absl::StatusOr<T>> {
-  using type = StatusOr<T>;
+  using type = absl::StatusOr<T>;
 };
 template <>
-struct AddStatus<Status> {
+struct AddStatus<absl::Status> {
-  using type = Status;
+  using type = absl::Status;
 };
 template <>
 struct AddStatus<void> {
-  using type = Status;
+  using type = absl::Status;
 };
 
 template <class R, class F, class... A>
@@ -282,7 +282,7 @@ struct CallAndAddStatusImpl {
 };
 template <class F, class... A>
 struct CallAndAddStatusImpl<void, F, A...> {
-  Status operator()(const F& f, A&&... a) {
+  absl::Status operator()(const F& f, A&&... a) {
     f(std::forward<A>(a)...);
     return {};
   }

mediapipe/framework/calculator_base_test.cc

@@ -183,8 +183,7 @@ TEST(CalculatorTest, CreateByNameWhitelisted) {
   CalculatorBaseRegistry::Register(
       "::mediapipe::test_ns::whitelisted_ns::DeadCalculator",
       absl::make_unique<internal::CalculatorBaseFactoryFor<
-          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>,
+          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>);
-      __FILE__, __LINE__);
 
   // A whitelisted calculator can be found in its own namespace.
   MP_EXPECT_OK(CalculatorBaseRegistry::CreateByNameInNamespace(  //

mediapipe/framework/calculator_context.h

@@ -109,9 +109,20 @@ class CalculatorContext {
   // use OutputStream::SetOffset() directly.
   void SetOffset(TimestampDiff offset);
 
-  // Returns the status of the graph run.
+  // DEPRECATED: This was intended to get graph run status during
+  // `CalculatorBase::Close` call. However, `Close` can run simultaneously with
+  // other calculators `CalculatorBase::Process`, hence the actual graph
+  // status may change any time and returned graph status here does not
+  // necessarily reflect the actual graph status.
   //
-  // NOTE: This method should only be called during CalculatorBase::Close().
+  // As an alternative, instead of checking graph status in `Close` and doing
+  // work for "done" state, you can enable timestamp bound processing for your
+  // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger
+  // `Process` on timestamp bound updates and handle "done" state there.
+  // Check examples in:
+  // mediapipe/framework/calculator_graph_summary_packet_test.cc.
+  //
+  ABSL_DEPRECATED("Does not reflect the actual graph status.")
   absl::Status GraphStatus() const { return graph_status_; }
 
   ProfilingContext* GetProfilingContext() const {

mediapipe/framework/calculator_graph.cc

@@ -839,6 +839,13 @@ absl::Status CalculatorGraph::PrepareForRun(
 }
 
 absl::Status CalculatorGraph::WaitUntilIdle() {
+  if (has_sources_) {
+    LOG_FIRST_N(WARNING, 1)
+        << "WaitUntilIdle called on a graph with source nodes, which "
+           "is not fully supported at the moment. Source nodes: "
+        << ListSourceNodes();
+  }
+
   MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle());
   VLOG(2) << "Scheduler idle.";
   absl::Status status = absl::OkStatus();
@@ -1368,6 +1375,16 @@ const OutputStreamManager* CalculatorGraph::FindOutputStreamManager(
               .get()[validated_graph_->OutputStreamIndex(name)];
 }
 
+std::string CalculatorGraph::ListSourceNodes() const {
+  std::vector<std::string> sources;
+  for (auto& node : nodes_) {
+    if (node->IsSource()) {
+      sources.push_back(node->DebugName());
+    }
+  }
+  return absl::StrJoin(sources, ", ");
+}
+
 namespace {
 void PrintTimingToInfo(const std::string& label, int64_t timer_value) {
   const int64_t total_seconds = timer_value / 1000000ll;

mediapipe/framework/calculator_graph.h

@@ -229,8 +229,11 @@ class CalculatorGraph {
   // Wait until the running graph is in the idle mode, which is when nothing can
   // be scheduled and nothing is running in the worker threads. This function
   // can be called only after StartRun().
+  //
   // NOTE: The graph must not have any source nodes because source nodes prevent
   // the running graph from becoming idle until the source nodes are done.
+  // Currently, `WaitUntilIdle` cannot be used reliably on graphs with any
+  // source nodes.
   absl::Status WaitUntilIdle();
 
   // Wait until a packet is emitted on one of the observed output streams.
@@ -594,6 +597,9 @@ class CalculatorGraph {
   // status before taking any action.
   void UpdateThrottledNodes(InputStreamManager* stream, bool* stream_was_full);
 
+  // Returns a comma-separated list of source nodes.
+  std::string ListSourceNodes() const;
+
 #if !MEDIAPIPE_DISABLE_GPU
   // Owns the legacy GpuSharedData if we need to create one for backwards
   // compatibility.

mediapipe/framework/calculator_graph_summary_packet_test.cc

@@ -0,0 +1,430 @@
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+using ::mediapipe::api2::Input;
+using ::mediapipe::api2::Node;
+using ::mediapipe::api2::Output;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsEmpty;
+using ::testing::Value;
+
+namespace {
+
+MATCHER_P2(IntPacket, value, timestamp, "") {
+  *result_listener << "where object is (value: " << arg.template Get<int>()
+                   << ", timestamp: " << arg.Timestamp() << ")";
+  return Value(arg.template Get<int>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(timestamp));
+}
+
+// Calculates and produces sum of all passed inputs when no more packets can be
+// expected on the input stream.
+class SummaryPacketCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"SUMMARY"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    // Makes sure there are no automatic timestamp bound updates when Process
+    // is called.
+    cc->SetTimestampOffset(TimestampDiff::Unset());
+    // Currently, only ImmediateInputStreamHandler supports "done" timestamp
+    // bound update. (ImmediateInputStreamhandler handles multiple input
+    // streams differently, so, in that case, calculator adjustments may be
+    // required.)
+    // TODO: update all input stream handlers to support "done"
+    // timestamp bound update.
+    cc->SetInputStreamHandler("ImmediateInputStreamHandler");
+    // Enables processing timestamp bound updates. For this use case we are
+    // specifically interested in "done" timestamp bound update. (E.g. when
+    // all input packet sources are closed.)
+    cc->SetProcessTimestampBounds(true);
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      value_ += kIn(cc).Get();
+      value_set_ = true;
+    }
+
+    if (kOut(cc).IsClosed()) {
+      // This can happen:
+      // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g.
+      //    source calculator finished generating packets sent to kIn) and
+      //    HasNextAllowedInStream() == true (which is an often case).
+      // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still
+      //    invoke Process() with Timestamp::Max to indicate "Done" timestamp
+      //    bound update.
+      return absl::OkStatus();
+    }
+
+    // TODO: input stream holding a packet with timestamp that has
+    // no next timestamp allowed in stream should always result in
+    // InputStream::IsDone() == true.
+    if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) {
+      // `Process` may or may not be invoked for "done" timestamp bound when
+      // upstream calculator fails in `Close`. Hence, extra care is needed to
+      // identify whether the calculator needs to send output.
+      // TODO: remove when "done" timestamp bound flakiness fixed.
+      if (value_set_) {
+        // kOut(cc).Send(value_) can be used here as well, however in the case
+        // of source calculator sending inputs into kIn the resulting timestamp
+        // is not well defined (e.g. it can be the last packet timestamp or
+        // Timestamp::Max())
+        // TODO: last packet from source should always result in
+        // InputStream::IsDone() == true.
+        kOut(cc).Send(value_, Timestamp::Max());
+      }
+      kOut(cc).Close();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  int value_ = 0;
+  bool value_set_ = false;
+};
+MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnClosingAllPacketSources) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp(11));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp::Max());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPreStreamTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PreStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPostStreamTimestamp) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input'
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: 'IN:input'
+          output_stream: 'SUMMARY:output'
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class IntGeneratorCalculator : public Node {
+ public:
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return tool::StatusStop();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnSourceCalculatorCompletion) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        node {
+          calculator: "IntGeneratorCalculator"
+          output_stream: "INT:int_value"
+        }
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: "IN:int_value"
+          output_stream: "SUMMARY:output"
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+class EmitOnCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnAnotherCalculatorClosure) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "EmitOnCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInClose) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInProcessCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInProcessCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInProcess) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInProcessCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/framework/deps/registration.h

@@ -16,7 +16,6 @@
 #define MEDIAPIPE_DEPS_REGISTRATION_H_
 
 #include <algorithm>
-#include <cstdint>
 #include <functional>
 #include <string>
 #include <tuple>
@@ -162,8 +161,7 @@ class FunctionRegistry {
   FunctionRegistry(const FunctionRegistry&) = delete;
   FunctionRegistry& operator=(const FunctionRegistry&) = delete;
 
-  RegistrationToken Register(absl::string_view name, Function func,
+  RegistrationToken Register(absl::string_view name, Function func)
-                             std::string filename, uint64_t line)
       ABSL_LOCKS_EXCLUDED(lock_) {
     std::string normalized_name = GetNormalizedName(name);
     absl::WriterMutexLock lock(&lock_);
@@ -173,21 +171,10 @@ class FunctionRegistry {
     }
     if (functions_.insert(std::make_pair(normalized_name, std::move(func)))
             .second) {
-#ifndef NDEBUG
-      locations_.emplace(normalized_name,
-                         std::make_pair(std::move(filename), line));
-#endif
       return RegistrationToken(
           [this, normalized_name]() { Unregister(normalized_name); });
     }
-#ifndef NDEBUG
-    LOG(FATAL) << "Function with name " << name << " already registered."
-               << " First registration at "
-               << locations_.at(normalized_name).first << ":"
-               << locations_.at(normalized_name).second;
-#else
     LOG(FATAL) << "Function with name " << name << " already registered.";
-#endif
     return RegistrationToken([]() {});
   }
 
@@ -316,11 +303,6 @@ class FunctionRegistry {
  private:
   mutable absl::Mutex lock_;
   absl::flat_hash_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
-#ifndef NDEBUG
-  // Stores filename and line number for useful debug log.
-  absl::flat_hash_map<std::string, std::pair<std::string, uint32_t>> locations_
-      ABSL_GUARDED_BY(lock_);
-#endif
 
   // For names included in NamespaceAllowlist, strips the namespace.
   std::string GetAdjustedName(absl::string_view name) {
@@ -351,10 +333,8 @@ class GlobalFactoryRegistry {
 
  public:
   static RegistrationToken Register(absl::string_view name,
-                                    typename Functions::Function func,
+                                    typename Functions::Function func) {
-                                    std::string filename, uint64_t line) {
+    return functions()->Register(name, std::move(func));
-    return functions()->Register(name, std::move(func), std::move(filename),
-                                 line);
   }
 
   // Invokes the specified factory function and returns the result.
@@ -414,12 +394,12 @@ class GlobalFactoryRegistry {
 #define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \
   static auto* REGISTRY_STATIC_VAR(registration_##name, __LINE__) =  \
       new mediapipe::RegistrationToken(                              \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
 
 #define REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, name, ...) \
   static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                       \
       new mediapipe::RegistrationToken(                                        \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
 
 }  // namespace mediapipe
 

mediapipe/framework/deps/safe_int.h

@@ -88,10 +88,13 @@ class SafeIntStrongIntValidator {
 
     // If the argument is floating point, we can do a simple check to make
     // sure the value is in range.  It is undefined behavior to convert to int
-    // from a float that is out of range.
+    // from a float that is out of range. Since large integers will loose some
+    // precision when being converted to floating point, the integer max and min
+    // are explicitly converted back to floating point for this comparison, in
+    // order to satisfy compiler warnings.
     if (std::is_floating_point<U>::value) {
-      if (arg < std::numeric_limits<T>::min() ||
+      if (arg < static_cast<U>(std::numeric_limits<T>::min()) ||
-          arg > std::numeric_limits<T>::max()) {
+          arg > static_cast<U>(std::numeric_limits<T>::max())) {
         ErrorType::Error("SafeInt: init from out of bounds float", arg, "=");
       }
     } else {
@@ -284,11 +287,11 @@ class SafeIntStrongIntValidator {
 // A SafeIntStrongIntValidator policy class to LOG(FATAL) on errors.
 struct LogFatalOnError {
   template <typename Tlhs, typename Trhs>
-  static void Error(const char *error, Tlhs lhs, Trhs rhs, const char *op) {
+  static void Error(const char* error, Tlhs lhs, Trhs rhs, const char* op) {
     LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")";
   }
   template <typename Tval>
-  static void Error(const char *error, Tval val, const char *op) {
+  static void Error(const char* error, Tval val, const char* op) {
     LOG(FATAL) << error << ": (" << op << val << ")";
   }
 };

mediapipe/framework/deps/status_builder.cc

@@ -68,11 +68,11 @@ StatusBuilder&& StatusBuilder::SetNoLogging() && {
   return std::move(SetNoLogging());
 }
 
-StatusBuilder::operator Status() const& {
+StatusBuilder::operator absl::Status() const& {
   return StatusBuilder(*this).JoinMessageToStatus();
 }
 
-StatusBuilder::operator Status() && { return JoinMessageToStatus(); }
+StatusBuilder::operator absl::Status() && { return JoinMessageToStatus(); }
 
 absl::Status StatusBuilder::JoinMessageToStatus() {
   if (!impl_) {

mediapipe/framework/deps/status_builder.h

@@ -83,8 +83,8 @@ class ABSL_MUST_USE_RESULT StatusBuilder {
     return std::move(*this << msg);
   }
 
-  operator Status() const&;
+  operator absl::Status() const&;
-  operator Status() &&;
+  operator absl::Status() &&;
 
   absl::Status JoinMessageToStatus();
 

mediapipe/framework/deps/strong_int.h

@@ -403,11 +403,11 @@ std::ostream &operator<<(std::ostream &os,
     lhs op## = rhs;                                                       \
     return lhs;                                                           \
   }
-STRONG_INT_VS_STRONG_INT_BINARY_OP(+);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(+)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(-);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(-)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(&);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(&)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(|);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(|)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(^)
 #undef STRONG_INT_VS_STRONG_INT_BINARY_OP
 
 // Define operators that take one StrongInt and one native integer argument.
@@ -431,12 +431,12 @@ STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
     rhs op## = lhs;                                                       \
     return rhs;                                                           \
   }
-STRONG_INT_VS_NUMERIC_BINARY_OP(*);
+STRONG_INT_VS_NUMERIC_BINARY_OP(*)
-NUMERIC_VS_STRONG_INT_BINARY_OP(*);
+NUMERIC_VS_STRONG_INT_BINARY_OP(*)
-STRONG_INT_VS_NUMERIC_BINARY_OP(/);
+STRONG_INT_VS_NUMERIC_BINARY_OP(/)
-STRONG_INT_VS_NUMERIC_BINARY_OP(%);
+STRONG_INT_VS_NUMERIC_BINARY_OP(%)
-STRONG_INT_VS_NUMERIC_BINARY_OP(<<);  // NOLINT(whitespace/operators)
+STRONG_INT_VS_NUMERIC_BINARY_OP(<<)  // NOLINT(whitespace/operators)
-STRONG_INT_VS_NUMERIC_BINARY_OP(>>);  // NOLINT(whitespace/operators)
+STRONG_INT_VS_NUMERIC_BINARY_OP(>>)  // NOLINT(whitespace/operators)
 #undef STRONG_INT_VS_NUMERIC_BINARY_OP
 #undef NUMERIC_VS_STRONG_INT_BINARY_OP
 
@@ -447,12 +447,12 @@ STRONG_INT_VS_NUMERIC_BINARY_OP(>>);  // NOLINT(whitespace/operators)
                           StrongInt<TagType, ValueType, ValidatorType> rhs) { \
     return lhs.value() op rhs.value();                                        \
   }
-STRONG_INT_COMPARISON_OP(==);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(==)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(!=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(!=)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(<);   // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(<)   // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(<=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(<=)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(>);   // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(>)   // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(>=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(>=)  // NOLINT(whitespace/operators)
 #undef STRONG_INT_COMPARISON_OP
 
 }  // namespace intops

mediapipe/framework/formats/BUILD

@@ -481,6 +481,7 @@ cc_library(
 cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":tensor",
         "//mediapipe/framework/port:gtest_main",

mediapipe/framework/formats/body_rig.proto

@@ -19,7 +19,7 @@ package mediapipe;
 // Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of
 // the joint and its visibility.
 message Joint {
-  // Joint rotation in 6D contineous representation ordered as
+  // Joint rotation in 6D continuous representation ordered as
   // [a1, b1, a2, b2, a3, b3].
   //
   // Such representation is more sutable for NN model training and can be

mediapipe/framework/formats/tensor.cc

@@ -117,8 +117,8 @@ MtlBufferView MtlBufferView::GetReadView(const Tensor& tensor,
       << "Tensor must be written prior to read from.";
   LOG_IF(FATAL,
          !(tensor.valid_ & (Tensor::kValidCpu | Tensor::kValidMetalBuffer)))
-      << "Tensor conversion between different GPU resources is not supported "
+      << "Tensor conversion between different GPU backing formats is not "
-         "yet.";
+         "supported yet.";
   auto lock(absl::make_unique<absl::MutexLock>(&tensor.view_mutex_));
   tensor.valid_ |= Tensor::kValidMetalBuffer;
   AllocateMtlBuffer(tensor, [command_buffer device]);
@@ -164,8 +164,8 @@ Tensor::OpenGlTexture2dView Tensor::GetOpenGlTexture2dReadView() const {
   LOG_IF(FATAL, valid_ == kValidNone)
       << "Tensor must be written prior to read from.";
   LOG_IF(FATAL, !(valid_ & (kValidCpu | kValidOpenGlTexture2d)))
-      << "Tensor conversion between different GPU resources is not supported "
+      << "Tensor conversion between different GPU backing formats is not "
-         "yet.";
+         "supported yet.";
   auto lock = absl::make_unique<absl::MutexLock>(&view_mutex_);
   AllocateOpenGlTexture2d();
   if (!(valid_ & kValidOpenGlTexture2d)) {
@@ -335,7 +335,8 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
                             kValidAHardwareBuffer |
 #endif  // MEDIAPIPE_TENSOR_USE_AHWB
                             kValidOpenGlBuffer)))
-      << "Tensor conversion between different GPU resources is not supported.";
+      << "Tensor conversion between different GPU backing formats is not "
+         "supported yet.";
   auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
   AllocateOpenGlBuffer();
   if (!(valid_ & kValidOpenGlBuffer)) {

mediapipe/framework/formats/tensor.h

@@ -117,11 +117,18 @@ class Tensor {
     Shape() = default;
     Shape(std::initializer_list<int> dimensions) : dims(dimensions) {}
     Shape(const std::vector<int>& dimensions) : dims(dimensions) {}
+    Shape(std::initializer_list<int> dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
+    Shape(const std::vector<int>& dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
     int num_elements() const {
       return std::accumulate(dims.begin(), dims.end(), 1,
                              std::multiplies<int>());
     }
     std::vector<int> dims;
+    // The Tensor has dynamic rather than static shape so the TFLite interpreter
+    // needs to be reallocated. Only relevant for CPU.
+    bool is_dynamic = false;
   };
   // Quantization parameters corresponding to the zero_point and scale value
   // made available by TfLite quantized (uint8/int8) tensors.

mediapipe/framework/formats/tensor_test.cc

@@ -2,6 +2,7 @@
 
 #include <cstring>
 #include <string>
+#include <vector>
 
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -34,6 +35,17 @@ TEST(General, TestDataTypes) {
   EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool));
 }
 
+TEST(General, TestDynamic) {
+  Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true));
+  EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4);
+  EXPECT_TRUE(t1.shape().is_dynamic);
+
+  std::vector<int> t2_dims = {4, 3, 2, 3};
+  Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true));
+  EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3);
+  EXPECT_TRUE(t2.shape().is_dynamic);
+}
+
 TEST(Cpu, TestMemoryAllocation) {
   Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3});
   auto v1 = t1.GetCpuWriteView();

mediapipe/framework/graph_service.h

@@ -44,7 +44,6 @@ class GraphServiceBase {
 
   constexpr GraphServiceBase(const char* key) : key(key) {}
 
-  virtual ~GraphServiceBase() = default;
   inline virtual absl::StatusOr<Packet> CreateDefaultObject() const {
     return DefaultInitializationUnsupported();
   }
@@ -52,14 +51,32 @@ class GraphServiceBase {
   const char* key;
 
  protected:
+  // `GraphService<T>` objects, deriving `GraphServiceBase` are designed to be
+  // global constants and not ever deleted through `GraphServiceBase`. Hence,
+  // protected and non-virtual destructor which helps to make `GraphService<T>`
+  // trivially destructible and properly defined as global constants.
+  //
+  // A class with any virtual functions should have a destructor that is either
+  // public and virtual or else protected and non-virtual.
+  // https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rc-dtor-virtual
+  ~GraphServiceBase() = default;
+
   absl::Status DefaultInitializationUnsupported() const {
     return absl::UnimplementedError(absl::StrCat(
         "Graph service '", key, "' does not support default initialization"));
   }
 };
 
+// A global constant to refer a service:
+// - Requesting `CalculatorContract::UseService` from calculator
+// - Accessing `Calculator/SubgraphContext::Service`from calculator/subgraph
+// - Setting before graph initialization `CalculatorGraph::SetServiceObject`
+//
+// NOTE: In headers, define your graph service reference safely as following:
+// `inline constexpr GraphService<YourService> kYourService("YourService");`
+//
 template <typename T>
-class GraphService : public GraphServiceBase {
+class GraphService final : public GraphServiceBase {
  public:
   using type = T;
   using packet_type = std::shared_ptr<T>;
@@ -68,7 +85,7 @@ class GraphService : public GraphServiceBase {
                                                  kDisallowDefaultInitialization)
       : GraphServiceBase(my_key), default_init_(default_init) {}
 
-  absl::StatusOr<Packet> CreateDefaultObject() const override {
+  absl::StatusOr<Packet> CreateDefaultObject() const final {
     if (default_init_ != kAllowDefaultInitialization) {
       return DefaultInitializationUnsupported();
     }

mediapipe/framework/graph_service_manager_test.cc

@@ -7,7 +7,7 @@
 
 namespace mediapipe {
 namespace {
-const GraphService<int> kIntService("mediapipe::IntService");
+constexpr GraphService<int> kIntService("mediapipe::IntService");
 }  // namespace
 
 TEST(GraphServiceManager, SetGetServiceObject) {

mediapipe/framework/graph_service_test.cc

@@ -14,6 +14,8 @@
 
 #include "mediapipe/framework/graph_service.h"
 
+#include <type_traits>
+
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
@@ -159,7 +161,7 @@ TEST_F(GraphServiceTest, CreateDefault) {
 
 struct TestServiceData {};
 
-const GraphService<TestServiceData> kTestServiceAllowDefaultInitialization(
+constexpr GraphService<TestServiceData> kTestServiceAllowDefaultInitialization(
     "kTestServiceAllowDefaultInitialization",
     GraphServiceBase::kAllowDefaultInitialization);
 
@@ -272,9 +274,13 @@ TEST(AllowDefaultInitializationGraphServiceTest,
                        HasSubstr("Service is unavailable.")));
 }
 
-const GraphService<TestServiceData> kTestServiceDisallowDefaultInitialization(
+constexpr GraphService<TestServiceData>
-    "kTestServiceDisallowDefaultInitialization",
+    kTestServiceDisallowDefaultInitialization(
-    GraphServiceBase::kDisallowDefaultInitialization);
+        "kTestServiceDisallowDefaultInitialization",
+        GraphServiceBase::kDisallowDefaultInitialization);
+
+static_assert(std::is_trivially_destructible_v<GraphService<TestServiceData>>,
+              "GraphService is not trivially destructible");
 
 class FailOnUnavailableOptionalDisallowDefaultInitServiceCalculator
     : public CalculatorBase {

mediapipe/framework/mediapipe_cc_test.bzl

@@ -15,11 +15,12 @@ def mediapipe_cc_test(
         platforms = ["linux", "android", "ios", "wasm"],
         exclude_platforms = None,
         # ios_unit_test arguments
-        ios_minimum_os_version = "11.0",
+        ios_minimum_os_version = "12.0",
         # android_cc_test arguments
         open_gl_driver = None,
         emulator_mini_boot = True,
         requires_full_emulation = True,
+        android_devices = {},
         # wasm_web_test arguments
         browsers = None,
         **kwargs):

mediapipe/framework/output_stream_shard.cc

@@ -94,7 +94,7 @@ const Packet& OutputStreamShard::Header() const {
 // binary.  This function can be defined in the .cc file because only two
 // versions are ever instantiated, and all call sites are within this .cc file.
 template <typename T>
-Status OutputStreamShard::AddPacketInternal(T&& packet) {
+absl::Status OutputStreamShard::AddPacketInternal(T&& packet) {
   if (IsClosed()) {
     return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC)
            << "Packet sent to closed stream \"" << Name() << "\".";
@@ -113,7 +113,7 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) {
            << timestamp.DebugString();
   }
 
-  Status result = output_stream_spec_->packet_type->Validate(packet);
+  absl::Status result = output_stream_spec_->packet_type->Validate(packet);
   if (!result.ok()) {
     return StatusBuilder(result, MEDIAPIPE_LOC).SetPrepend() << absl::StrCat(
                "Packet type mismatch on calculator outputting to stream \"",
@@ -132,14 +132,14 @@ Status OutputStreamShard::AddPacketInternal(T&& packet) {
 }
 
 void OutputStreamShard::AddPacket(const Packet& packet) {
-  Status status = AddPacketInternal(packet);
+  absl::Status status = AddPacketInternal(packet);
   if (!status.ok()) {
     output_stream_spec_->TriggerErrorCallback(status);
   }
 }
 
 void OutputStreamShard::AddPacket(Packet&& packet) {
-  Status status = AddPacketInternal(std::move(packet));
+  absl::Status status = AddPacketInternal(std::move(packet));
   if (!status.ok()) {
     output_stream_spec_->TriggerErrorCallback(status);
   }

mediapipe/framework/packet.h

@@ -466,8 +466,7 @@ struct MessageRegistrationImpl {
 template <typename T>
 NoDestructor<mediapipe::RegistrationToken>
     MessageRegistrationImpl<T>::registration(MessageHolderRegistry::Register(
-        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder,
+        T{}.GetTypeName(), MessageRegistrationImpl<T>::CreateMessageHolder));
-        __FILE__, __LINE__));
 
 // For non-Message payloads, this does nothing.
 template <typename T, typename Enable = void>

mediapipe/framework/port/opencv_photo_inc.h

@@ -0,0 +1,20 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+#define MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_
+
+#include "third_party/OpenCV/photo.hpp"
+
+#endif  // MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_

mediapipe/framework/scheduler.cc

@@ -273,8 +273,8 @@ absl::Status Scheduler::WaitForObservedOutput() {
 // Idleness requires:
 // 1. either the graph has no source nodes or all source nodes are closed, and
 // 2. no packets are added to graph input streams.
-// For simplicity, we only allow WaitUntilIdle() to be called on a graph with
+// For simplicity, we only fully support WaitUntilIdle() to be called on a graph
-// no source nodes. (This is enforced by CalculatorGraph::WaitUntilIdle().)
+// with no source nodes.
 // The application must ensure no other threads are adding packets to graph
 // input streams while a WaitUntilIdle() call is in progress.
 absl::Status Scheduler::WaitUntilIdle() {

mediapipe/framework/subgraph.cc

@@ -64,13 +64,13 @@ GraphRegistry::GraphRegistry(
 void GraphRegistry::Register(
     const std::string& type_name,
     std::function<std::unique_ptr<Subgraph>()> factory) {
-  local_factories_.Register(type_name, factory, __FILE__, __LINE__);
+  local_factories_.Register(type_name, factory);
 }
 
 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphConfig& config) {
-  Register(type_name, [config] {
+  local_factories_.Register(type_name, [config] {
     auto result = absl::make_unique<ProtoSubgraph>(config);
     return std::unique_ptr<Subgraph>(result.release());
   });
@@ -79,7 +79,7 @@ void GraphRegistry::Register(const std::string& type_name,
 // TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphTemplate& templ) {
-  Register(type_name, [templ] {
+  local_factories_.Register(type_name, [templ] {
     auto result = absl::make_unique<TemplateSubgraph>(templ);
     return std::unique_ptr<Subgraph>(result.release());
   });

mediapipe/framework/test_service.cc

@@ -16,15 +16,6 @@
 
 namespace mediapipe {
 
-const GraphService<TestServiceObject> kTestService(
-    "test_service", GraphServiceBase::kDisallowDefaultInitialization);
-const GraphService<int> kAnotherService(
-    "another_service", GraphServiceBase::kAllowDefaultInitialization);
-const GraphService<NoDefaultConstructor> kNoDefaultService(
-    "no_default_service", GraphServiceBase::kAllowDefaultInitialization);
-const GraphService<NeedsCreateMethod> kNeedsCreateService(
-    "needs_create_service", GraphServiceBase::kAllowDefaultInitialization);
-
 absl::Status TestServiceCalculator::GetContract(CalculatorContract* cc) {
   cc->Inputs().Index(0).Set<int>();
   cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0));

mediapipe/framework/test_service.h

@@ -22,14 +22,17 @@ namespace mediapipe {
 
 using TestServiceObject = std::map<std::string, int>;
 
-extern const GraphService<TestServiceObject> kTestService;
+inline constexpr GraphService<TestServiceObject> kTestService(
-extern const GraphService<int> kAnotherService;
+    "test_service", GraphServiceBase::kDisallowDefaultInitialization);
+inline constexpr GraphService<int> kAnotherService(
+    "another_service", GraphServiceBase::kAllowDefaultInitialization);
 
 class NoDefaultConstructor {
  public:
   NoDefaultConstructor() = delete;
 };
-extern const GraphService<NoDefaultConstructor> kNoDefaultService;
+inline constexpr GraphService<NoDefaultConstructor> kNoDefaultService(
+    "no_default_service", GraphServiceBase::kAllowDefaultInitialization);
 
 class NeedsCreateMethod {
  public:
@@ -40,7 +43,8 @@ class NeedsCreateMethod {
  private:
   NeedsCreateMethod() = default;
 };
-extern const GraphService<NeedsCreateMethod> kNeedsCreateService;
+inline constexpr GraphService<NeedsCreateMethod> kNeedsCreateService(
+    "needs_create_service", GraphServiceBase::kAllowDefaultInitialization);
 
 // Use a service.
 class TestServiceCalculator : public CalculatorBase {

mediapipe/framework/timestamp.cc

@@ -131,6 +131,13 @@ Timestamp Timestamp::NextAllowedInStream() const {
   return *this + 1;
 }
 
+bool Timestamp::HasNextAllowedInStream() const {
+  if (*this >= Max() || *this == PreStream()) {
+    return false;
+  }
+  return true;
+}
+
 Timestamp Timestamp::PreviousAllowedInStream() const {
   if (*this <= Min() || *this == PostStream()) {
     // Indicates that no previous timestamps may occur.

mediapipe/framework/timestamp.h

@@ -57,7 +57,7 @@ namespace mediapipe {
 // have underflow/overflow etc.  This type is used internally by Timestamp
 // and TimestampDiff.
 MEDIAPIPE_DEFINE_SAFE_INT_TYPE(TimestampBaseType, int64,
-                               mediapipe::intops::LogFatalOnError);
+                               mediapipe::intops::LogFatalOnError)
 
 class TimestampDiff;
 
@@ -186,6 +186,10 @@ class Timestamp {
   // CHECKs that this->IsAllowedInStream().
   Timestamp NextAllowedInStream() const;
 
+  // Returns true if there's a next timestamp in the range [Min .. Max] after
+  // this one.
+  bool HasNextAllowedInStream() const;
+
   // Returns the previous timestamp in the range [Min .. Max], or
   // Unstarted() if no Packets may preceed one with this timestamp.
   Timestamp PreviousAllowedInStream() const;

mediapipe/framework/timestamp_test.cc

@@ -125,6 +125,22 @@ TEST(TimestampTest, NextAllowedInStream) {
             Timestamp::PostStream().NextAllowedInStream());
 }
 
+TEST(TimestampTest, HasNextAllowedInStream) {
+  EXPECT_TRUE(Timestamp::Min().HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Min() + 1).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(-1000).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(0).HasNextAllowedInStream());
+  EXPECT_TRUE(Timestamp(1000).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 2).HasNextAllowedInStream());
+  EXPECT_TRUE((Timestamp::Max() - 1).HasNextAllowedInStream());
+
+  EXPECT_FALSE(Timestamp::PreStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Max().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::PostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::OneOverPostStream().HasNextAllowedInStream());
+  EXPECT_FALSE(Timestamp::Done().HasNextAllowedInStream());
+}
+
 TEST(TimestampTest, SpecialValueDifferences) {
   {  // Lower range
     const std::vector<Timestamp> timestamps = {

mediapipe/framework/tool/BUILD

@@ -530,6 +530,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],