Ensure that -std=c++14/17 is the first argument passed to Glog

PiperOrigin-RevId: 552509553

README.md

@@ -1,17 +1,16 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
 title: Home
 nav_order: 1
 ---
 
 ----
 
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
 [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
 as the primary developer documentation site for MediaPipe as of April 3, 2023.*
 
-*This notice and web page will be removed on June 1, 2023.*
-
 ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
 
 **Attention**: MediaPipe Solutions Preview is an early release. [Learn

WORKSPACE

@@ -45,12 +45,13 @@ http_archive(
 )
 
 http_archive(
-   name = "rules_foreign_cc",
+    name = "rules_foreign_cc",
-   strip_prefix = "rules_foreign_cc-0.1.0",
+    sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
-   url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
+    strip_prefix = "rules_foreign_cc-0.9.0",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
 )
 
-load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
 
 rules_foreign_cc_dependencies()
 
@@ -156,22 +157,22 @@ http_archive(
 # 2020-08-21
 http_archive(
     name = "com_github_glog_glog",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
+    strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
     urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
     ],
 )
 http_archive(
     name = "com_github_glog_glog_no_gflags",
-    strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
+    strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
-    sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+    sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
     build_file = "@//third_party:glog_no_gflags.BUILD",
     urls = [
-        "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+        "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
     ],
     patches = [
-        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
+        "@//third_party:com_github_glog_glog.diff",
     ],
     patch_args = [
         "-p1",
@@ -266,10 +267,10 @@ http_archive(
 
 http_archive(
     name = "com_googlesource_code_re2",
-    sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
+    sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
-    strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
+    strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
     urls = [
-        "https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
+        "https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
     ],
 )
 
@@ -484,9 +485,10 @@ http_archive(
 )
 
 # TensorFlow repo should always go after the other external dependencies.
-# TF on 2023-04-12.
+# TF on 2023-06-13.
-_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
+_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
-_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
+# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
+_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
 http_archive(
     name = "org_tensorflow",
     urls = [

docs/index.md

@@ -1,17 +1,16 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
 title: Home
 nav_order: 1
 ---
 
 ----
 
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
 [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
 as the primary developer documentation site for MediaPipe as of April 3, 2023.*
 
-*This notice and web page will be removed on June 1, 2023.*
-
 ![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
 
 **Attention**: MediaPipe Solutions Preview is an early release. [Learn

docs/solutions/face_detection.md

@@ -20,9 +20,9 @@ nav_order: 1
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
 site.*
 
 ----

docs/solutions/face_mesh.md

@@ -20,9 +20,9 @@ nav_order: 2
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
 site.*
 
 ----

docs/solutions/iris.md

@@ -20,9 +20,9 @@ nav_order: 3
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
 site.*
 
 ----

docs/solutions/pose.md

@@ -22,9 +22,9 @@ nav_order: 5
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
 site.*
 
 ----

docs/solutions/pose_classification.md

@@ -21,7 +21,7 @@ nav_order: 1
 ---
 
 **Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
 Solution. For more information, see the
 [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
 site.*

docs/solutions/solutions.md

@@ -1,5 +1,6 @@
 ---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe/solutions/guide#legacy
 title: MediaPipe Legacy Solutions
 nav_order: 3
 has_children: true
@@ -13,8 +14,7 @@ has_toc: false
 {:toc}
 ---
 
-**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
+**Attention:** *We have ended support for
-ended support for
 [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
 as of March 1, 2023. All other
 [MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
 Solutions at:
 [https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
 
-*This notice and web page will be removed on June 1, 2023.*
-
-----
-
-<br><br><br><br><br><br><br><br><br><br>
-<br><br><br><br><br><br><br><br><br><br>
-<br><br><br><br><br><br><br><br><br><br>
-
 ----
 
 MediaPipe offers open source cross-platform, customizable ML solutions for live

mediapipe/BUILD

@@ -68,30 +68,108 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-# Note: this cannot just match "apple_platform_type": "macos" because that option
+# Generic MacOS.
-# defaults to "macos" even when building on Linux!
+config_setting(
-alias(
     name = "macos",
-    actual = select({
+    constraint_values = [
-        ":macos_i386": ":macos_i386",
+        "@platforms//os:macos",
-        ":macos_x86_64": ":macos_x86_64",
+    ],
-        ":macos_arm64": ":macos_arm64",
-        "//conditions:default": ":macos_i386",  # Arbitrarily chosen from above.
-    }),
     visibility = ["//visibility:public"],
 )
 
-# Note: this also matches on crosstool_top so that it does not produce ambiguous
+# MacOS x86 64-bit.
-# selectors when used together with "android".
+config_setting(
+    name = "macos_x86_64",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:x86_64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# MacOS ARM64.
+config_setting(
+    name = "macos_arm64",
+    constraint_values = [
+        "@platforms//os:macos",
+        "@platforms//cpu:arm64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# Generic iOS.
 config_setting(
     name = "ios",
-    values = {
+    constraint_values = [
-        "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
+        "@platforms//os:ios",
-        "apple_platform_type": "ios",
+    ],
-    },
     visibility = ["//visibility:public"],
 )
 
+# iOS device ARM32.
+config_setting(
+    name = "ios_armv7",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64.
+config_setting(
+    name = "ios_arm64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64E.
+config_setting(
+    name = "ios_arm64e",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64e",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 32-bit.
+config_setting(
+    name = "ios_i386",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:x86_32",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 64-bit.
+config_setting(
+    name = "ios_x86_64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:x86_64",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# iOS simulator ARM64.
+config_setting(
+    name = "ios_sim_arm64",
+    constraint_values = [
+        "@platforms//os:ios",
+        "@platforms//cpu:arm64",
+        "@build_bazel_apple_support//constraints:simulator",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# Generic Apple.
 alias(
     name = "apple",
     actual = select({
@@ -102,49 +180,6 @@ alias(
     visibility = ["//visibility:public"],
 )
 
-config_setting(
-    name = "macos_i386",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin",
-    },
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "macos_x86_64",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin_x86_64",
-    },
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "macos_arm64",
-    values = {
-        "apple_platform_type": "macos",
-        "cpu": "darwin_arm64",
-    },
-    visibility = ["//visibility:public"],
-)
-
-[
-    config_setting(
-        name = arch,
-        values = {"cpu": arch},
-        visibility = ["//visibility:public"],
-    )
-    for arch in [
-        "ios_i386",
-        "ios_x86_64",
-        "ios_armv7",
-        "ios_arm64",
-        "ios_arm64e",
-        "ios_sim_arm64",
-    ]
-]
-
 config_setting(
     name = "windows",
     values = {"cpu": "x64_windows"},

mediapipe/calculators/audio/BUILD

@@ -219,12 +219,10 @@ cc_library(
     deps = [
         ":time_series_framer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/framework/port:integral_types",
-        "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
         "//mediapipe/util:time_series_util",
         "@com_google_audio_tools//audio/dsp:window_functions",
         "@eigen_archive//:eigen3",
@@ -319,6 +317,20 @@ cc_test(
     ],
 )
 
+cc_binary(
+    name = "time_series_framer_calculator_benchmark",
+    srcs = ["time_series_framer_calculator_benchmark.cc"],
+    deps = [
+        ":time_series_framer_calculator",
+        ":time_series_framer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:time_series_header_cc_proto",
+        "@com_google_benchmark//:benchmark",
+    ],
+)
+
 cc_test(
     name = "time_series_framer_calculator_test",
     srcs = ["time_series_framer_calculator_test.cc"],

mediapipe/calculators/audio/spectrogram_calculator.cc

@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
 // Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
 const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
 
+namespace {
+std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
+    const SpectrogramCalculatorOptions::WindowType window_type) {
+  switch (window_type) {
+    // The cosine window and square root of Hann are equivalent.
+    case SpectrogramCalculatorOptions::COSINE:
+    case SpectrogramCalculatorOptions::SQRT_HANN:
+      return std::make_unique<audio_dsp::CosineWindow>();
+    case SpectrogramCalculatorOptions::HANN:
+      return std::make_unique<audio_dsp::HannWindow>();
+    case SpectrogramCalculatorOptions::HAMMING:
+      return std::make_unique<audio_dsp::HammingWindow>();
+  }
+  return nullptr;
+}
+}  // namespace
+
 absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
   SpectrogramCalculatorOptions spectrogram_options =
       cc->Options<SpectrogramCalculatorOptions>();
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
 
   output_scale_ = spectrogram_options.output_scale();
 
-  std::vector<double> window;
+  auto window_fun = MakeWindowFun(spectrogram_options.window_type());
-  switch (spectrogram_options.window_type()) {
+  if (window_fun == nullptr) {
-    case SpectrogramCalculatorOptions::COSINE:
+    return absl::Status(absl::StatusCode::kInvalidArgument,
-      audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
+                        absl::StrCat("Invalid window type ",
-                                                   &window);
+                                     spectrogram_options.window_type()));
-      break;
-    case SpectrogramCalculatorOptions::HANN:
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      break;
-    case SpectrogramCalculatorOptions::HAMMING:
-      audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                    &window);
-      break;
-    case SpectrogramCalculatorOptions::SQRT_HANN: {
-      audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
-                                                 &window);
-      absl::c_transform(window, window.begin(),
-                        [](double x) { return std::sqrt(x); });
-      break;
-    }
   }
+  std::vector<double> window;
+  window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
 
   // Propagate settings down to the actual Spectrogram object.
   spectrogram_generators_.clear();

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
     HANN = 0;
     HAMMING = 1;
     COSINE = 2;
-    SQRT_HANN = 4;
+    SQRT_HANN = 4;  // Alias of COSINE.
   }
   optional WindowType window_type = 6 [default = HANN];
 

mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -15,9 +15,7 @@
 // Defines TimeSeriesFramerCalculator.
 #include <math.h>
 
-#include <deque>
+#include <vector>
-#include <memory>
-#include <string>
 
 #include "Eigen/Core"
 #include "audio/dsp/window_functions.h"
@@ -25,9 +23,8 @@
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
 #include "mediapipe/util/time_series_util.h"
 
 namespace mediapipe {
@@ -88,11 +85,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   absl::Status Close(CalculatorContext* cc) override;
 
  private:
-  // Adds input data to the internal buffer.
-  void EnqueueInput(CalculatorContext* cc);
-  // Constructs and emits framed output packets.
-  void FrameOutput(CalculatorContext* cc);
-
   Timestamp CurrentOutputTimestamp() {
     if (use_local_timestamp_) {
       return current_timestamp_;
@@ -106,14 +98,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
                  Timestamp::kTimestampUnitsPerSecond);
   }
 
-  // Returns the timestamp of a sample on a base, which is usually the time
-  // stamp of a packet.
-  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
-                                   int64_t number_of_samples) {
-    return timestamp_base + round(number_of_samples / sample_rate_ *
-                                  Timestamp::kTimestampUnitsPerSecond);
-  }
-
   // The number of input samples to advance after the current output frame is
   // emitted.
   int next_frame_step_samples() const {
@@ -142,61 +126,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   Timestamp initial_input_timestamp_;
   // The current timestamp is updated along with the incoming packets.
   Timestamp current_timestamp_;
-  int num_channels_;
 
-  // Each entry in this deque consists of a single sample, i.e. a
+  // Samples are buffered in a vector of sample blocks.
-  // single column vector, and its timestamp.
+  class SampleBlockBuffer {
-  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
+   public:
+    // Initializes the buffer.
+    void Init(double sample_rate, int num_channels) {
+      ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
+      num_channels_ = num_channels;
+      num_samples_ = 0;
+      first_block_offset_ = 0;
+    }
+
+    // Number of channels, equal to the number of rows in each Matrix.
+    int num_channels() const { return num_channels_; }
+    // Total number of available samples over all blocks.
+    int num_samples() const { return num_samples_; }
+
+    // Pushes a new block of samples on the back of the buffer with `timestamp`
+    // being the input timestamp of the packet containing the Matrix.
+    void Push(const Matrix& samples, Timestamp timestamp);
+    // Copies `count` samples from the front of the buffer. If there are fewer
+    // samples than this, the result is zero padded to have `count` samples.
+    // The timestamp of the last copied sample is written to *last_timestamp.
+    // This output is used below to update `current_timestamp_`, which is only
+    // used when `use_local_timestamp` is true.
+    Matrix CopySamples(int count, Timestamp* last_timestamp) const;
+    // Drops `count` samples from the front of the buffer. If `count` exceeds
+    // `num_samples()`, the buffer is emptied.  Returns how many samples were
+    // dropped.
+    int DropSamples(int count);
+
+   private:
+    struct Block {
+      // Matrix of num_channels rows by num_samples columns, a block of possibly
+      // multiple samples.
+      Matrix samples;
+      // Timestamp of the first sample in the Block. This comes from the input
+      // packet's timestamp that contains this Matrix.
+      Timestamp timestamp;
+
+      Block() : timestamp(Timestamp::Unstarted()) {}
+      Block(const Matrix& samples, Timestamp timestamp)
+          : samples(samples), timestamp(timestamp) {}
+      int num_samples() const { return samples.cols(); }
+    };
+    std::vector<Block> blocks_;
+    // Number of timestamp units per sample. Used to compute timestamps as
+    // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
+    double ts_units_per_sample_;
+    // Number of rows in each Matrix.
+    int num_channels_;
+    // The total number of samples over all blocks, equal to
+    // (sum_i blocks_[i].num_samples()) - first_block_offset_.
+    int num_samples_;
+    // The number of samples in the first block that have been discarded. This
+    // way we can cheaply represent "partially discarding" a block.
+    int first_block_offset_;
+  } sample_buffer_;
 
   bool use_window_;
-  Matrix window_;
+  Eigen::RowVectorXf window_;
 
   bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
 
-void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
+void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
-  const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
+                                                         Timestamp timestamp) {
-
+  num_samples_ += samples.cols();
-  for (int i = 0; i < input_frame.cols(); ++i) {
+  blocks_.emplace_back(samples, timestamp);
-    sample_buffer_.emplace_back(std::make_pair(
-        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
-  }
 }
 
-void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
+Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
-  while (sample_buffer_.size() >=
+    int count, Timestamp* last_timestamp) const {
+  Matrix copied(num_channels_, count);
+
+  if (!blocks_.empty()) {
+    int num_copied = 0;
+    // First block has an offset for samples that have been discarded.
+    int offset = first_block_offset_;
+    int n;
+    Timestamp last_block_ts;
+    int last_sample_index;
+
+    for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
+      n = std::min(it->num_samples() - offset, count);
+      // Copy `n` samples from the next block.
+      copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
+      count -= n;
+      num_copied += n;
+      last_block_ts = it->timestamp;
+      last_sample_index = offset + n - 1;
+      offset = 0;  // No samples have been discarded in subsequent blocks.
+    }
+
+    // Compute the timestamp of the last copied sample.
+    *last_timestamp =
+        last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
+  }
+
+  if (count > 0) {
+    copied.rightCols(count).setZero();  // Zero pad if needed.
+  }
+
+  return copied;
+}
+
+int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
+  if (blocks_.empty()) {
+    return 0;
+  }
+
+  auto block_it = blocks_.begin();
+  if (first_block_offset_ + count < block_it->num_samples()) {
+    // `count` is less than the remaining samples in the first block.
+    first_block_offset_ += count;
+    num_samples_ -= count;
+    return count;
+  }
+
+  int num_samples_dropped = block_it->num_samples() - first_block_offset_;
+  count -= num_samples_dropped;
+  first_block_offset_ = 0;
+
+  for (++block_it; block_it != blocks_.end(); ++block_it) {
+    if (block_it->num_samples() > count) {
+      break;
+    }
+    num_samples_dropped += block_it->num_samples();
+    count -= block_it->num_samples();
+  }
+
+  blocks_.erase(blocks_.begin(), block_it);  // Drop whole blocks.
+  if (!blocks_.empty()) {
+    first_block_offset_ = count;  // Drop part of the next block.
+    num_samples_dropped += count;
+  }
+
+  num_samples_ -= num_samples_dropped;
+  return num_samples_dropped;
+}
+
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+    initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
+  }
+
+  // Add input data to the internal buffer.
+  sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
+                      cc->InputTimestamp());
+
+  // Construct and emit framed output packets.
+  while (sample_buffer_.num_samples() >=
          frame_duration_samples_ + samples_still_to_drop_) {
-    while (samples_still_to_drop_ > 0) {
+    sample_buffer_.DropSamples(samples_still_to_drop_);
-      sample_buffer_.pop_front();
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
-      --samples_still_to_drop_;
+                                                     &current_timestamp_);
-    }
     const int frame_step_samples = next_frame_step_samples();
-    std::unique_ptr<Matrix> output_frame(
+    samples_still_to_drop_ = frame_step_samples;
-        new Matrix(num_channels_, frame_duration_samples_));
-    for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
-         ++i) {
-      output_frame->col(i) = sample_buffer_.front().first;
-      current_timestamp_ = sample_buffer_.front().second;
-      sample_buffer_.pop_front();
-    }
-    const int frame_overlap_samples =
-        frame_duration_samples_ - frame_step_samples;
-    if (frame_overlap_samples > 0) {
-      for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
-        current_timestamp_ = sample_buffer_[i].second;
-      }
-    } else {
-      samples_still_to_drop_ = -frame_overlap_samples;
-    }
 
     if (use_window_) {
-      *output_frame = (output_frame->array() * window_.array()).matrix();
+      // Apply the window to each row of output_frame.
+      output_frame.array().rowwise() *= window_.array();
     }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
-                               CurrentOutputTimestamp());
+                                         .At(CurrentOutputTimestamp()));
     ++cumulative_output_frames_;
     cumulative_completed_samples_ += frame_step_samples;
   }
@@ -206,35 +303,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
     // fact to enable packet queueing optimizations.
     cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
   }
-}
-
-absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
-  if (initial_input_timestamp_ == Timestamp::Unstarted()) {
-    initial_input_timestamp_ = cc->InputTimestamp();
-    current_timestamp_ = initial_input_timestamp_;
-  }
-
-  EnqueueInput(cc);
-  FrameOutput(cc);
 
   return absl::OkStatus();
 }
 
 absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
-  while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
+  sample_buffer_.DropSamples(samples_still_to_drop_);
-    sample_buffer_.pop_front();
-    --samples_still_to_drop_;
-  }
-  if (!sample_buffer_.empty() && pad_final_packet_) {
-    std::unique_ptr<Matrix> output_frame(new Matrix);
-    output_frame->setZero(num_channels_, frame_duration_samples_);
-    for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i].first;
-      current_timestamp_ = sample_buffer_[i].second;
-    }
 
-    cc->Outputs().Index(0).Add(output_frame.release(),
+  if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
-                               CurrentOutputTimestamp());
+    Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+                                                     &current_timestamp_);
+    cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
+                                         .At(CurrentOutputTimestamp()));
   }
 
   return absl::OkStatus();
@@ -258,7 +338,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
       cc->Inputs().Index(0).Header(), &input_header));
 
   sample_rate_ = input_header.sample_rate();
-  num_channels_ = input_header.num_channels();
+  sample_buffer_.Init(sample_rate_, input_header.num_channels());
   frame_duration_samples_ = time_series_util::SecondsToSamples(
       framer_options.frame_duration_seconds(), sample_rate_);
   RET_CHECK_GT(frame_duration_samples_, 0)
@@ -312,9 +392,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   }
 
   if (use_window_) {
-    window_ = Matrix::Ones(num_channels_, 1) *
+    window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
-              Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
+                                             frame_duration_samples_)
-                                          frame_duration_samples_)
                   .cast<float>();
   }
   use_local_timestamp_ = framer_options.use_local_timestamp();

mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc

@@ -0,0 +1,92 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Benchmark for TimeSeriesFramerCalculator.
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/packet.h"
+
+using ::mediapipe::Matrix;
+
+void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
+  constexpr float kSampleRate = 32000.0;
+  constexpr int kNumChannels = 2;
+  constexpr int kFrameDurationSeconds = 5.0;
+  std::mt19937 rng(0 /*seed*/);
+  // Input around a half second's worth of samples at a time.
+  std::uniform_int_distribution<int> input_size_dist(15000, 17000);
+  // Generate a pool of random blocks of samples up front.
+  std::vector<Matrix> sample_pool;
+  sample_pool.reserve(20);
+  for (int i = 0; i < 20; ++i) {
+    sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
+  }
+  std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
+
+  mediapipe::CalculatorGraphConfig config;
+  config.add_input_stream("input");
+  config.add_output_stream("output");
+  auto* node = config.add_node();
+  node->set_calculator("TimeSeriesFramerCalculator");
+  node->add_input_stream("input");
+  node->add_output_stream("output");
+  mediapipe::TimeSeriesFramerCalculatorOptions* options =
+      node->mutable_options()->MutableExtension(
+          mediapipe::TimeSeriesFramerCalculatorOptions::ext);
+  options->set_frame_duration_seconds(kFrameDurationSeconds);
+
+  for (auto _ : state) {
+    state.PauseTiming();  // Pause benchmark timing.
+
+    // Prepare input packets of random blocks of samples.
+    std::vector<mediapipe::Packet> input_packets;
+    input_packets.reserve(32);
+    float t = 0;
+    for (int i = 0; i < 32; ++i) {
+      auto samples =
+          std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
+      const int num_samples = samples->cols();
+      input_packets.push_back(mediapipe::Adopt(samples.release())
+                                  .At(mediapipe::Timestamp::FromSeconds(t)));
+      t += num_samples / kSampleRate;
+    }
+    // Initialize graph.
+    mediapipe::CalculatorGraph graph;
+    CHECK_OK(graph.Initialize(config));
+    // Prepare input header.
+    auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
+    header->set_sample_rate(kSampleRate);
+    header->set_num_channels(kNumChannels);
+
+    state.ResumeTiming();  // Resume benchmark timing.
+
+    CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
+    for (auto& packet : input_packets) {
+      CHECK_OK(graph.AddPacketToInputStream("input", packet));
+    }
+    CHECK(!graph.HasError());
+    CHECK_OK(graph.CloseAllInputStreams());
+    CHECK_OK(graph.WaitUntilIdle());
+  }
+}
+BENCHMARK(BM_TimeSeriesFramerCalculator);
+
+BENCHMARK_MAIN();

mediapipe/calculators/core/BUILD

@@ -117,6 +117,7 @@ mediapipe_proto_library(
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/framework/formats:classification_proto",
         "//mediapipe/framework/formats:landmark_proto",
+        "//mediapipe/framework/formats:matrix_data_proto",
         "//mediapipe/framework/formats:time_series_header_proto",
     ],
 )
@@ -192,17 +193,19 @@ cc_library(
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/gpu:gpu_buffer",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
     ],
     alwayslink = 1,
 )
@@ -215,18 +218,18 @@ cc_library(
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
-        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/framework/port:status",
+        "//mediapipe/gpu:gpu_buffer",
         "//mediapipe/util:render_data_cc_proto",
+        "@com_google_absl//absl/status",
         "@org_tensorflow//tensorflow/lite:framework",
     ],
     alwayslink = 1,
@@ -287,6 +290,7 @@ cc_library(
         "//mediapipe/framework/api2:node",
         "//mediapipe/framework/api2:port",
         "//mediapipe/framework/formats:classification_cc_proto",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:integral_types",
@@ -295,8 +299,7 @@ cc_library(
         "//mediapipe/util:render_data_cc_proto",
         "@org_tensorflow//tensorflow/lite:framework",
     ] + select({
-        "//mediapipe/gpu:disable_gpu": [],
+        ":ios_or_disable_gpu": [],
-        "//mediapipe:ios": [],
         "//conditions:default": [
             "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
         ],
@@ -378,17 +381,6 @@ cc_library(
     alwayslink = 1,
 )
 
-cc_library(
-    name = "clip_detection_vector_size_calculator",
-    srcs = ["clip_detection_vector_size_calculator.cc"],
-    deps = [
-        ":clip_vector_size_calculator",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:detection_cc_proto",
-    ],
-    alwayslink = 1,
-)
-
 cc_test(
     name = "clip_vector_size_calculator_test",
     srcs = ["clip_vector_size_calculator_test.cc"],
@@ -904,6 +896,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
@@ -1136,6 +1129,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework/port:status",
     ],
     alwayslink = 1,
@@ -1164,6 +1158,7 @@ cc_library(
         "//mediapipe/framework:collection_item_id",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix_data_cc_proto",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -1238,6 +1233,7 @@ cc_library(
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ],

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -17,10 +17,13 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
 
 namespace mediapipe {
 
@@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
 typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
 REGISTER_CALCULATOR(BeginLoopTensorCalculator);
 
+// A calculator to process std::vector<mediapipe::ImageFrame>.
+typedef BeginLoopCalculator<std::vector<ImageFrame>>
+    BeginLoopImageFrameCalculator;
+REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
+
+// A calculator to process std::vector<mediapipe::GpuBuffer>.
+typedef BeginLoopCalculator<std::vector<GpuBuffer>>
+    BeginLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
+
+// A calculator to process std::vector<mediapipe::Image>.
+typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
+REGISTER_CALCULATOR(BeginLoopImageCalculator);
+
+// A calculator to process std::vector<float>.
+typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
+REGISTER_CALCULATOR(BeginLoopFloatCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.h

@@ -15,47 +15,57 @@
 #ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
 
+#include "absl/status/status.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/status_macros.h"
 
 namespace mediapipe {
 
 // Calculator for implementing loops on iterable collections inside a MediaPipe
-// graph.
+// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
+// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
+// First, instantiate specializations in the loop calculators' implementations
+// if missing:
+//   BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
+//   EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
+// Then, the following graph transforms an item of type InputIterT to an
+// OutputIterT by applying InputToOutputConverter to every element:
 //
-// It is designed to be used like:
+// node {                                        # Type        @timestamp
-//
+//   calculator:    "BeginLoopInputTCalculator"
-// node {
+//   input_stream:  "ITERABLE:input_iterable"    # InputIterT  @iterable_ts
-//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "CLONE:extra_input"          # ExtraT      @extra_ts
-//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_iterator"        # InputT      @loop_internal_ts
-//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "CLONE:cloned_extra_input"   # ExtraT      @loop_internal_ts
-//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "BATCH_END:iterable_ts"      # Timestamp   @loop_internal_ts
 // }
 //
 // node {
-//   calculator:    "ElementToBlaConverterSubgraph"
+//   calculator:    "InputToOutputConverter"
-//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   input_stream:  "INPUT:input_iterator"       # InputT      @loop_internal_ts
-//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+//   input_stream:  "EXTRA:cloned_extra_input"   # ExtraT      @loop_internal_ts
+//   output_stream: "OUTPUT:output_iterator"     # OutputT     @loop_internal_ts
 // }
 //
 // node {
-//   calculator:    "EndLoopWithOutputCalculator"
+//   calculator:    "EndLoopOutputTCalculator"
-//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "ITEM:output_iterator"       # OutputT     @loop_internal_ts
-//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   input_stream:  "BATCH_END:iterable_ts"      # Timestamp   @loop_internal_ts
-//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
+//   output_stream: "ITERABLE:output_iterable"   # OutputIterT @iterable_ts
 // }
 //
+// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
+// The output packets of this calculator are part of the loop body and have
+// loop-internal timestamps that are unrelated to the input iterator timestamp.
+//
 // Input streams tagged with "CLONE" are cloned to the corresponding output
-// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
+// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
-// can run multiple times, once per element in the "ITERABLE" for each pakcet
+// sub-graph can run multiple times, once per element in the "ITERABLE" for each
-// clone of the packets in the "CLONE" input streams.
+// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
+// inputs as loop-wide constants.
 template <typename IterableT>
 class BeginLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
 typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
 
+typedef ConcatenateVectorCalculator<std::string>
+    ConcatenateStringVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"
@@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
     ConcatenateRenderDataVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
 
+typedef ConcatenateVectorCalculator<mediapipe::Image>
+    ConcatenateImageVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
 }  // namespace mediapipe

mediapipe/calculators/core/concatenate_vector_calculator_test.cc

@@ -30,13 +30,15 @@ namespace mediapipe {
 typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
 MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
 
-void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
+template <typename T>
+void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
                     CalculatorRunner* runner) {
   runner->MutableInputs()->Index(index).packets.push_back(
-      MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
+      MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
 }
 
-void AddInputVectors(const std::vector<std::vector<int>>& inputs,
+template <typename T>
+void AddInputVectors(const std::vector<std::vector<T>>& inputs,
                      int64_t timestamp, CalculatorRunner* runner) {
   for (int i = 0; i < inputs.size(); ++i) {
     AddInputVector(i, inputs[i], timestamp, runner);
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
   EXPECT_EQ(0, outputs.size());
 }
 
+TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
+  CalculatorRunner runner("ConcatenateStringVectorCalculator",
+                          /*options_string=*/"", /*num_inputs=*/3,
+                          /*num_outputs=*/1, /*num_side_packets=*/0);
+
+  std::vector<std::vector<std::string>> inputs = {
+      {"a", "b"}, {"c"}, {"d", "e", "f"}};
+  AddInputVectors(inputs, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
+  EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
+}
+
 typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
     TestConcatenateUniqueIntPtrCalculator;
 MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);

mediapipe/calculators/core/constant_side_packet_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
 #include "mediapipe/framework/formats/time_series_header.pb.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/integral_types.h"
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
         packet.Set<LandmarkList>();
       } else if (packet_options.has_double_value()) {
         packet.Set<double>();
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set<MatrixData>();
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set<TimeSeriesHeader>();
+      } else if (packet_options.has_int64_value()) {
+        packet.Set<int64_t>();
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
             MakePacket<LandmarkList>(packet_options.landmark_list_value()));
       } else if (packet_options.has_double_value()) {
         packet.Set(MakePacket<double>(packet_options.double_value()));
+      } else if (packet_options.has_matrix_data_value()) {
+        packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
       } else if (packet_options.has_time_series_header_value()) {
         packet.Set(MakePacket<TimeSeriesHeader>(
             packet_options.time_series_header_value()));
+      } else if (packet_options.has_int64_value()) {
+        packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
       } else {
         return absl::InvalidArgumentError(
             "None of supported values were specified in options.");

mediapipe/calculators/core/constant_side_packet_calculator.proto

@@ -19,6 +19,7 @@ package mediapipe;
 import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/formats/classification.proto";
 import "mediapipe/framework/formats/landmark.proto";
+import "mediapipe/framework/formats/matrix_data.proto";
 import "mediapipe/framework/formats/time_series_header.proto";
 
 message ConstantSidePacketCalculatorOptions {
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
   message ConstantSidePacket {
     oneof value {
       int32 int_value = 1;
+      uint64 uint64_value = 5;
+      int64 int64_value = 11;
       float float_value = 2;
+      double double_value = 9;
       bool bool_value = 3;
       string string_value = 4;
-      uint64 uint64_value = 5;
       ClassificationList classification_list_value = 6;
       LandmarkList landmark_list_value = 7;
-      double double_value = 9;
       TimeSeriesHeader time_series_header_value = 10;
+      MatrixData matrix_data_value = 12;
     }
   }
 

mediapipe/calculators/core/constant_side_packet_calculator_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdint>
 #include <string>
 
 #include "absl/strings/string_view.h"
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
   DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
   DoTestSingleSidePacket("{ bool_value: true }", true);
   DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
+  DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
 }
 
 TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {

mediapipe/calculators/core/end_loop_calculator.cc

@@ -19,10 +19,12 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/util/render_data.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
@@ -68,8 +70,18 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
 typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
 REGISTER_CALCULATOR(EndLoopTensorCalculator);
 
+typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
+REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
+
+typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
+
 typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
     EndLoopImageCalculator;
 REGISTER_CALCULATOR(EndLoopImageCalculator);
 
+typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
+    EndLoopAffineMatrixCalculator;
+REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.h

@@ -17,13 +17,11 @@
 
 #include <type_traits>
 
+#include "absl/status/status.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/port/integral_types.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
 
 namespace mediapipe {
 
@@ -33,27 +31,7 @@ namespace mediapipe {
 // from the "BATCH_END" tagged input stream, it emits the aggregated results
 // at the original timestamp contained in the "BATCH_END" input stream.
 //
-// It is designed to be used like:
+// See BeginLoopCalculator for a usage example.
-//
-// node {
-//   calculator:    "BeginLoopWithIterableCalculator"
-//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
-//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
-//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-// }
-//
-// node {
-//   calculator:    "ElementToBlaConverterSubgraph"
-//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
-//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
-// }
-//
-// node {
-//   calculator:    "EndLoopWithOutputCalculator"
-//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
-//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
-//   output_stream: "ITERABLE:aggregated_result"   # IterableU @ext_ts
-// }
 template <typename IterableT>
 class EndLoopCalculator : public CalculatorBase {
   using ItemT = typename IterableT::value_type;
@@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
       }
       // Try to consume the item and move it into the collection. If the items
       // are not consumable, then try to copy them instead. If the items are
-      // not copiable, then an error will be returned.
+      // not copyable, then an error will be returned.
       auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
       if (item_ptr_or.ok()) {
         input_stream_collection_->push_back(std::move(*item_ptr_or.value()));

mediapipe/calculators/core/get_vector_item_calculator.cc

@@ -17,6 +17,7 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
 
 namespace mediapipe {
 namespace api2 {
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
     GetVectorItemCalculator<mediapipe::Detection>;
 REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
 
+using GetNormalizedRectVectorItemCalculator =
+    GetVectorItemCalculator<NormalizedRect>;
+REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
+
+using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
+REGISTER_CALCULATOR(GetRectVectorItemCalculator);
+
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/core/previous_loopback_calculator.cc

@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
           // However, LOOP packet is empty.
           kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
         } else {
-          kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          // Avoids sending leftovers to a stream that's already closed.
+          if (!kPrevLoop(cc).IsClosed()) {
+            kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+          }
         }
         loop_packets_.pop_front();
         main_packet_specs_.pop_front();

mediapipe/calculators/core/split_vector_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
 typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
 REGISTER_CALCULATOR(SplitFloatVectorCalculator);
 
+typedef SplitVectorCalculator<mediapipe::Image, false>
+    SplitImageVectorCalculator;
+REGISTER_CALCULATOR(SplitImageVectorCalculator);
+
+typedef SplitVectorCalculator<std::array<float, 16>, false>
+    SplitAffineMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/stream_to_side_packet_calculator.cc

@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/timestamp.h"
 
 namespace mediapipe {
+namespace api2 {
 
 // A calculator that takes a packet of an input stream and converts it to an
 // output side packet. This calculator only works under the assumption that the
@@ -28,21 +30,21 @@ namespace mediapipe {
 //   input_stream: "stream"
 //   output_side_packet: "side_packet"
 // }
-class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+class StreamToSidePacketCalculator : public Node {
  public:
-  static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
+  static constexpr Input<AnyType>::Optional kIn{""};
-    cc->Inputs().Index(0).SetAny();
+  static constexpr SideOutput<SameType<kIn>> kOut{""};
-    cc->OutputSidePackets().Index(0).SetAny();
+
-    return absl::OkStatus();
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
-  }
 
   absl::Status Process(mediapipe::CalculatorContext* cc) override {
-    mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
+    kOut(cc).Set(
-    cc->OutputSidePackets().Index(0).Set(
+        kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
-        packet.At(mediapipe::Timestamp::Unset()));
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(StreamToSidePacketCalculator);
 
+MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -135,7 +135,6 @@ cc_library(
     deps = [
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/port:opencv_imgcodecs",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:status",
     ],
@@ -317,6 +316,7 @@ cc_library(
 cc_test(
     name = "image_cropping_calculator_test",
     srcs = ["image_cropping_calculator_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":image_cropping_calculator",
         ":image_cropping_calculator_cc_proto",
@@ -650,6 +650,7 @@ cc_library(
 cc_test(
     name = "segmentation_smoothing_calculator_test",
     srcs = ["segmentation_smoothing_calculator_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":image_clone_calculator",
         ":image_clone_calculator_cc_proto",
@@ -771,7 +772,10 @@ cc_test(
         "//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
         "//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
     ],
-    tags = ["desktop_only_test"],
+    tags = [
+        "desktop_only_test",
+        "not_run:arm",
+    ],
     deps = [
         ":affine_transformation",
         ":image_transformation_calculator",

mediapipe/calculators/image/bilateral_filter_calculator.cc

@@ -112,7 +112,7 @@ class BilateralFilterCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(BilateralFilterCalculator);
 
 absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
 
   if (cc->Inputs().HasTag(kInputFrameTag) &&
       cc->Inputs().HasTag(kInputFrameTagGpu)) {

mediapipe/calculators/image/segmentation_smoothing_calculator.cc

@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
 
 absl::Status SegmentationSmoothingCalculator::GetContract(
     CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
 
   cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
   cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(SetAlphaCalculator);
 
 absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
 
   bool use_gpu = false;
 

mediapipe/calculators/image/yuv_to_image_calculator.cc

@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
   buf[0] = (fourcc >> 24) & 0xff;
   buf[1] = (fourcc >> 16) & 0xff;
   buf[2] = (fourcc >> 8) & 0xff;
-  buf[3] = (fourcc)&0xff;
+  buf[3] = (fourcc) & 0xff;
   buf[4] = 0;
   return std::string(buf);
 }

mediapipe/calculators/tensor/BUILD

@@ -228,7 +228,6 @@ cc_library(
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
     ],
     alwayslink = 1,
@@ -280,7 +279,6 @@ cc_library(
         "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
         "//mediapipe/tasks/metadata:metadata_schema_cc",
         "@com_google_absl//absl/status",
-        "@com_google_absl//absl/status:statusor",
     ],
     alwayslink = 1,
 )
@@ -394,7 +392,7 @@ mediapipe_proto_library(
 # If you want to have precise control of which implementations to include (e.g. for strict binary
 # size concerns), depend on those implementations directly, and do not depend on
 # :inference_calculator.
-# In all cases, use "InferenceCalulator" in your graphs.
+# In all cases, use "InferenceCalculator" in your graphs.
 cc_library_with_tflite(
     name = "inference_calculator_interface",
     srcs = ["inference_calculator.cc"],
@@ -655,6 +653,11 @@ cc_library(
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
         "//conditions:default": ["tensor_converter_calculator_gpu_deps"],
+    }) + select({
+        "//mediapipe:apple": [
+            "//third_party/apple_frameworks:MetalKit",
+        ],
+        "//conditions:default": [],
     }),
     alwayslink = 1,
 )
@@ -1052,6 +1055,7 @@ cc_test(
         "testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
         "testdata/image_to_tensor/noop_except_range.png",
     ],
+    tags = ["not_run:arm"],
     deps = [
         ":image_to_tensor_calculator",
         ":image_to_tensor_converter",

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -282,18 +282,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
   if (options.has_volume_gain_db()) {
     gain_ = pow(10, options.volume_gain_db() / 20.0);
   }
-  RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+  if (options.has_source_sample_rate()) {
-            !kAudioIn(cc).Header().IsEmpty())
+    source_sample_rate_ = options.source_sample_rate();
-      << "Must either specify the time series header of the \"AUDIO\" stream "
+  } else {
-         "or have the \"SAMPLE_RATE\" stream connected.";
+    RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
-  if (!kAudioIn(cc).Header().IsEmpty()) {
+              !kAudioIn(cc).Header().IsEmpty())
-    mediapipe::TimeSeriesHeader input_header;
+        << "Must either specify the time series header of the \"AUDIO\" stream "
-    MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+           "or have the \"SAMPLE_RATE\" stream connected.";
-        kAudioIn(cc).Header(), &input_header));
+    if (!kAudioIn(cc).Header().IsEmpty()) {
-    if (stream_mode_) {
+      mediapipe::TimeSeriesHeader input_header;
-      MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      MP_RETURN_IF_ERROR(
-    } else {
+          mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
-      source_sample_rate_ = input_header.sample_rate();
+              kAudioIn(cc).Header(), &input_header));
+      if (stream_mode_) {
+        MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+      } else {
+        source_sample_rate_ = input_header.sample_rate();
+      }
     }
   }
   AppendZerosToSampleBuffer(padding_samples_before_);

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
   // The volume gain, measured in dB.
   // Scale the input audio amplitude by 10^(volume_gain_db/20).
   optional double volume_gain_db = 12;
+
+  // The source number of samples per second (hertz) of the input audio buffers.
+  optional double source_sample_rate = 13;
 }

mediapipe/calculators/tensor/bert_preprocessor_calculator.cc

@@ -22,7 +22,6 @@
 
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
@@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
   input_tensors.reserve(kNumInputTensorsForBert);
   for (int i = 0; i < kNumInputTensorsForBert; ++i) {
     input_tensors.push_back(
-        {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
+        {Tensor::ElementType::kInt32,
+         Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
   }
   std::memcpy(input_tensors[input_ids_tensor_index_]
                   .GetCpuWriteView()

mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc

@@ -69,6 +69,7 @@ class InferenceCalculatorGlAdvancedImpl
             gpu_delegate_options);
     absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
     absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
+    bool UseSerializedModel() const { return use_serialized_model_; }
 
    private:
     bool use_kernel_caching_ = false;
@@ -150,8 +151,6 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
 }
 
 absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
-  MP_RETURN_IF_ERROR(
-      on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
   return gpu_helper_.RunInGlContext([this]() -> absl::Status {
     tflite_gpu_runner_.reset();
     return absl::OkStatus();
@@ -226,9 +225,14 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
                          tflite_gpu_runner_->GetOutputShapes()[i].c};
   }
 
+  if (on_disk_cache_helper_.UseSerializedModel()) {
+    tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel();
+  }
+
   MP_RETURN_IF_ERROR(
       on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
-  return tflite_gpu_runner_->Build();
+  MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
+  return on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get());
 }
 
 #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)

mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc

@@ -96,6 +96,19 @@ absl::StatusOr<std::vector<Tensor>> InferenceInterpreterDelegateRunner::Run(
     CalculatorContext* cc, const std::vector<Tensor>& input_tensors) {
   // Read CPU input into tensors.
   RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size());
+
+  // If the input tensors have dynamic shape, then the tensors need to be
+  // resized and reallocated before we can copy the tensor values.
+  bool resized_tensor_shapes = false;
+  for (int i = 0; i < input_tensors.size(); ++i) {
+    if (input_tensors[i].shape().is_dynamic) {
+      interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims);
+      resized_tensor_shapes = true;
+    }
+  }
+  // Reallocation is needed for memory sanity.
+  if (resized_tensor_shapes) interpreter_->AllocateTensors();
+
   for (int i = 0; i < input_tensors.size(); ++i) {
     const TfLiteType input_tensor_type =
         interpreter_->tensor(interpreter_->inputs()[i])->type;

mediapipe/calculators/tensor/regex_preprocessor_calculator.cc

@@ -20,7 +20,6 @@
 #include <vector>
 
 #include "absl/status/status.h"
-#include "absl/status/statusor.h"
 #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h"
 #include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/api2/port.h"
@@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) {
   // not found in the tokenizer vocab.
   std::vector<Tensor> result;
   result.push_back(
-      {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})});
+      {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})});
   std::memcpy(result[0].GetCpuWriteView().buffer<int32_t>(),
               input_tokens.data(), input_tokens.size() * sizeof(int32_t));
   kTensorsOut(cc).Send(std::move(result));

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -256,6 +256,7 @@ class TensorsToDetectionsCalculator : public Node {
 
   bool gpu_inited_ = false;
   bool gpu_input_ = false;
+  bool gpu_has_enough_work_groups_ = true;
   bool anchors_init_ = false;
 };
 MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator);
@@ -291,7 +292,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) {
 absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
   auto output_detections = absl::make_unique<std::vector<Detection>>();
   bool gpu_processing = false;
-  if (CanUseGpu()) {
+  if (CanUseGpu() && gpu_has_enough_work_groups_) {
     // Use GPU processing only if at least one input tensor is already on GPU
     // (to avoid CPU->GPU overhead).
     for (const auto& tensor : *kInTensors(cc)) {
@@ -321,11 +322,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
     RET_CHECK(!has_custom_box_indices_);
   }
 
-  if (gpu_processing) {
+  if (gpu_processing && !gpu_inited_) {
-    if (!gpu_inited_) {
+    auto status = GpuInit(cc);
-      MP_RETURN_IF_ERROR(GpuInit(cc));
+    if (status.ok()) {
       gpu_inited_ = true;
+    } else if (status.code() == absl::StatusCode::kFailedPrecondition) {
+      // For initialization error because of hardware limitation, fallback to
+      // CPU processing.
+      LOG(WARNING) << status.message();
+    } else {
+      // For other error, let the error propagates.
+      return status;
     }
+  }
+  if (gpu_processing && gpu_inited_) {
     MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get()));
   } else {
     MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get()));
@@ -346,17 +356,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
     // TODO: Add flexible input tensor size handling.
     auto raw_box_tensor =
         &input_tensors[tensor_mapping_.detections_tensor_index()];
-    RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3);
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
     RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options";
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+    if (raw_box_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+      // The tensors from CPU inference has dim 3.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_);
+    } else if (raw_box_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of box Tensor must be 3 or 4.");
+    }
     auto raw_score_tensor =
         &input_tensors[tensor_mapping_.scores_tensor_index()];
-    RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3);
+    if (raw_score_tensor->shape().dims.size() == 3) {
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      // The tensors from CPU inference has dim 3.
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
-    RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_);
+    } else if (raw_score_tensor->shape().dims.size() == 4) {
+      // The tensors from GPU inference has dim 4. For gpu-cpu fallback support,
+      // we allow tensors with 4 dims.
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_);
+      RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_);
+    } else {
+      return absl::InvalidArgumentError(
+          "The dimensions of score Tensor must be 3 or 4.");
+    }
     auto raw_box_view = raw_box_tensor->GetCpuReadView();
     auto raw_boxes = raw_box_view.buffer<float>();
     auto raw_scores_view = raw_score_tensor->GetCpuReadView();
@@ -1111,8 +1145,13 @@ void main() {
     int max_wg_size;  //  typically <= 1024
     glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
                     &max_wg_size);  // y-dim
-    CHECK_LT(num_classes_, max_wg_size)
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
-        << "# classes must be < " << max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
     // TODO support better filtering.
     if (class_index_set_.is_allowlist) {
       CHECK_EQ(class_index_set_.values.size(),
@@ -1370,7 +1409,13 @@ kernel void scoreKernel(
         Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2});
     // # filter classes supported is hardware dependent.
     int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup;
-    CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
+    gpu_has_enough_work_groups_ = num_classes_ < max_wg_size;
+    if (!gpu_has_enough_work_groups_) {
+      return absl::FailedPreconditionError(absl::StrFormat(
+          "Hardware limitation: Processing will be done on CPU, because "
+          "num_classes %d exceeds the max work_group size %d.",
+          num_classes_, max_wg_size));
+    }
   }
 
 #endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)

mediapipe/calculators/tensorflow/BUILD

@@ -406,8 +406,13 @@ cc_library(
     alwayslink = 1,
 )
 
-# This dependency removed tensorflow_jellyfish_deps and xprofilez_with_server because they failed
+# This dependency removed the following 3 targets because they failed Boq conformance test:
-# Boq conformance test. Weigh your use case to see if this will work for you.
+#
+# tensorflow_jellyfish_deps
+# jfprof_lib
+# xprofilez_with_server
+#
+# If you need them plz consider tensorflow_inference_calculator_no_envelope_loader.
 cc_library(
     name = "tensorflow_inference_calculator_for_boq",
     srcs = ["tensorflow_inference_calculator.cc"],
@@ -927,7 +932,6 @@ cc_test(
         "//mediapipe/framework:timestamp",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:image_frame",
-        "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/formats:location",
         "//mediapipe/framework/formats:location_opencv",
         "//mediapipe/framework/port:gtest_main",
@@ -1077,6 +1081,7 @@ cc_test(
     linkstatic = 1,
     deps = [
         ":tensor_to_image_frame_calculator",
+        ":tensor_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:image_frame",

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -164,8 +164,8 @@ class PackMediaSequenceCalculator : public CalculatorBase {
       }
     }
 
-    CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
+    RET_CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
-          cc->OutputSidePackets().HasTag(kSequenceExampleTag))
+              cc->OutputSidePackets().HasTag(kSequenceExampleTag))
         << "Neither the output stream nor the output side packet is set to "
            "output the sequence example.";
     if (cc->Outputs().HasTag(kSequenceExampleTag)) {

mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc

@@ -23,7 +23,6 @@
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/formats/location.h"
 #include "mediapipe/framework/formats/location_opencv.h"
 #include "mediapipe/framework/port/gmock.h"
@@ -96,7 +95,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
   mpms::SetClipMediaId(test_video_id, input_sequence.get());
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   encoded_image.set_width(2);
@@ -139,7 +139,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
   mpms::SetClipMediaId(test_video_id, input_sequence.get());
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   encoded_image.set_width(2);
@@ -378,7 +379,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
       Adopt(input_sequence.release());
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   auto image_ptr =
@@ -410,7 +412,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
 
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   std::string test_flow_string(bytes.begin(), bytes.end());
   OpenCvImageEncoderCalculatorResults encoded_flow;
   encoded_flow.set_encoded_image(test_flow_string);
@@ -618,7 +621,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
   }
   cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   encoded_image.set_width(width);
@@ -767,7 +771,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
 
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   std::string test_flow_string(bytes.begin(), bytes.end());
   OpenCvImageEncoderCalculatorResults encoded_flow;
   encoded_flow.set_encoded_image(test_flow_string);
@@ -813,7 +818,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
   mpms::SetClipMediaId(test_video_id, input_sequence.get());
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   std::string test_flow_string(bytes.begin(), bytes.end());
   OpenCvImageEncoderCalculatorResults encoded_flow;
   encoded_flow.set_encoded_image(test_flow_string);
@@ -970,7 +976,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
   auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   encoded_image.set_width(2);
@@ -1021,7 +1028,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
   auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
   cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
   std::vector<uchar> bytes;
-  ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
+  ASSERT_TRUE(
+      cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
   OpenCvImageEncoderCalculatorResults encoded_image;
   encoded_image.set_encoded_image(bytes.data(), bytes.size());
   int height = 2;

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc

@@ -65,6 +65,7 @@ class TensorToImageFrameCalculator : public CalculatorBase {
 
  private:
   float scale_factor_;
+  bool scale_per_frame_min_max_;
 };
 
 REGISTER_CALCULATOR(TensorToImageFrameCalculator);
@@ -88,6 +89,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) {
 absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) {
   scale_factor_ =
       cc->Options<TensorToImageFrameCalculatorOptions>().scale_factor();
+  scale_per_frame_min_max_ = cc->Options<TensorToImageFrameCalculatorOptions>()
+                                 .scale_per_frame_min_max();
   cc->SetOffset(TimestampDiff(0));
   return absl::OkStatus();
 }
@@ -109,16 +112,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
   auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8);
   const int32_t total_size = height * width * depth;
 
+  if (scale_per_frame_min_max_) {
+    RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT)
+        << "Setting scale_per_frame_min_max requires FLOAT input tensors.";
+  }
   ::std::unique_ptr<const ImageFrame> output;
   if (input_tensor.dtype() == tensorflow::DT_FLOAT) {
     // Allocate buffer with alignments.
     std::unique_ptr<uint8_t[]> buffer(
         new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]);
     auto data = input_tensor.flat<float>().data();
+    float min = 1e23;
+    float max = -1e23;
+    if (scale_per_frame_min_max_) {
+      for (int i = 0; i < total_size; ++i) {
+        float d = scale_factor_ * data[i];
+        if (d < min) {
+          min = d;
+        }
+        if (d > max) {
+          max = d;
+        }
+      }
+    }
     for (int i = 0; i < total_size; ++i) {
-      float d = scale_factor_ * data[i];
+      float d = data[i];
-      if (d < 0) d = 0;
+      if (scale_per_frame_min_max_) {
-      if (d > 255) d = 255;
+        d = 255 * (d - min) / (max - min + 1e-9);
+      } else {
+        d = scale_factor_ * d;
+        if (d < 0) d = 0;
+        if (d > 255) d = 255;
+      }
       buffer[i] = d;
     }
     output = ::absl::make_unique<ImageFrame>(

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto

@@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions {
   // Multiples floating point tensor outputs by this value before converting to
   // uint8. This is useful for converting from range [0, 1] to [0, 255]
   optional float scale_factor = 1 [default = 1.0];
+
+  // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255]
+  // per frame. This overrides any explicit scale_factor.
+  optional bool scale_per_frame_min_max = 2 [default = false];
 }

mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc

@@ -11,7 +11,9 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <type_traits>
 
+#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE";
 template <class TypeParam>
 class TensorToImageFrameCalculatorTest : public ::testing::Test {
  protected:
-  void SetUpRunner() {
+  void SetUpRunner(bool scale_per_frame_min_max = false) {
     CalculatorGraphConfig::Node config;
     config.set_calculator("TensorToImageFrameCalculator");
     config.add_input_stream("TENSOR:input_tensor");
     config.add_output_stream("IMAGE:output_image");
+    config.mutable_options()
+        ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext)
+        ->set_scale_per_frame_min_max(scale_per_frame_min_max);
     runner_ = absl::make_unique<CalculatorRunner>(config);
   }
 
@@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest,
   }
 }
 
+TYPED_TEST(TensorToImageFrameCalculatorTest,
+           Converts3DTensorToImageFrame2DGrayWithScaling) {
+  this->SetUpRunner(true);
+  auto& runner = this->runner_;
+  constexpr int kWidth = 16;
+  constexpr int kHeight = 8;
+  const tf::TensorShape tensor_shape{kHeight, kWidth};
+  auto tensor = absl::make_unique<tf::Tensor>(
+      tf::DataTypeToEnum<TypeParam>::v(), tensor_shape);
+  auto tensor_vec = tensor->template flat<TypeParam>().data();
+
+  // Writing sequence of integers as floats which we want normalized.
+  tensor_vec[0] = 255;
+  for (int i = 1; i < kWidth * kHeight; ++i) {
+    tensor_vec[i] = 200;
+  }
+
+  const int64_t time = 1234;
+  runner->MutableInputs()->Tag(kTensor).packets.push_back(
+      Adopt(tensor.release()).At(Timestamp(time)));
+
+  if (!std::is_same<TypeParam, float>::value) {
+    EXPECT_FALSE(runner->Run().ok());
+    return;  // Short circuit because does not apply to other types.
+  } else {
+    EXPECT_TRUE(runner->Run().ok());
+    const std::vector<Packet>& output_packets =
+        runner->Outputs().Tag(kImage).packets;
+    EXPECT_EQ(1, output_packets.size());
+    EXPECT_EQ(time, output_packets[0].Timestamp().Value());
+    const ImageFrame& output_image = output_packets[0].Get<ImageFrame>();
+    EXPECT_EQ(ImageFormat::GRAY8, output_image.Format());
+    EXPECT_EQ(kWidth, output_image.Width());
+    EXPECT_EQ(kHeight, output_image.Height());
+
+    EXPECT_EQ(255, output_image.PixelData()[0]);
+    for (int i = 1; i < kWidth * kHeight; ++i) {
+      const uint8_t pixel_value = output_image.PixelData()[i];
+      ASSERT_EQ(0, pixel_value);
+    }
+  }
+}
+
 }  // namespace mediapipe

mediapipe/calculators/util/BUILD

@@ -899,16 +899,77 @@ mediapipe_proto_library(
 cc_library(
     name = "landmarks_smoothing_calculator",
     srcs = ["landmarks_smoothing_calculator.cc"],
+    hdrs = ["landmarks_smoothing_calculator.h"],
     deps = [
         ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "landmarks_smoothing_calculator_utils",
+    srcs = ["landmarks_smoothing_calculator_utils.cc"],
+    hdrs = ["landmarks_smoothing_calculator_utils.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util/filtering:one_euro_filter",
         "//mediapipe/util/filtering:relative_velocity_filter",
-        "@com_google_absl//absl/algorithm:container",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "landmarks_smoothing_calculator_utils_test",
+    size = "small",
+    srcs = ["landmarks_smoothing_calculator_utils_test.cc"],
+    deps = [
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+    ],
+)
+
+cc_library(
+    name = "multi_landmarks_smoothing_calculator",
+    srcs = ["multi_landmarks_smoothing_calculator.cc"],
+    hdrs = ["multi_landmarks_smoothing_calculator.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "multi_world_landmarks_smoothing_calculator",
+    srcs = ["multi_world_landmarks_smoothing_calculator.cc"],
+    hdrs = ["multi_world_landmarks_smoothing_calculator.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -172,7 +172,7 @@ class AnnotationOverlayCalculator : public CalculatorBase {
 REGISTER_CALCULATOR(AnnotationOverlayCalculator);
 
 absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
-  CHECK_GE(cc->Inputs().NumEntries(), 1);
+  RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
 
   bool use_gpu = false;
 
@@ -189,13 +189,13 @@ absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
 #if !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kGpuBufferTag)) {
     cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
-    CHECK(cc->Outputs().HasTag(kGpuBufferTag));
+    RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
     use_gpu = true;
   }
 #endif  // !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kImageFrameTag)) {
     cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    CHECK(cc->Outputs().HasTag(kImageFrameTag));
+    RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
   }
 
   // Data streams to render.

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2020 The MediaPipe Authors.
+// Copyright 2023 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,471 +12,105 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.h"
+
 #include <memory>
 
-#include "absl/algorithm/container.h"
 #include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
-#include "mediapipe/util/filtering/one_euro_filter.h"
-#include "mediapipe/util/filtering/relative_velocity_filter.h"
 
 namespace mediapipe {
+namespace api2 {
 
 namespace {
 
-constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
-constexpr char kLandmarksTag[] = "LANDMARKS";
-constexpr char kImageSizeTag[] = "IMAGE_SIZE";
-constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI";
-constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
-constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
-
 using ::mediapipe::NormalizedRect;
-using mediapipe::OneEuroFilter;
 using ::mediapipe::Rect;
-using mediapipe::RelativeVelocityFilter;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
-
+using ::mediapipe::landmarks_smoothing::InitializeLandmarksFilter;
-void NormalizedLandmarksToLandmarks(
+using ::mediapipe::landmarks_smoothing::LandmarksFilter;
-    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
-    const int image_height, LandmarkList* landmarks) {
+using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
-  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
-    const auto& norm_landmark = norm_landmarks.landmark(i);
-
-    auto* landmark = landmarks->add_landmark();
-    landmark->set_x(norm_landmark.x() * image_width);
-    landmark->set_y(norm_landmark.y() * image_height);
-    // Scale Z the same way as X (using image width).
-    landmark->set_z(norm_landmark.z() * image_width);
-    landmark->set_visibility(norm_landmark.visibility());
-    landmark->set_presence(norm_landmark.presence());
-  }
-}
-
-void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
-                                    const int image_width,
-                                    const int image_height,
-                                    NormalizedLandmarkList* norm_landmarks) {
-  for (int i = 0; i < landmarks.landmark_size(); ++i) {
-    const auto& landmark = landmarks.landmark(i);
-
-    auto* norm_landmark = norm_landmarks->add_landmark();
-    norm_landmark->set_x(landmark.x() / image_width);
-    norm_landmark->set_y(landmark.y() / image_height);
-    // Scale Z the same way as X (using image width).
-    norm_landmark->set_z(landmark.z() / image_width);
-    norm_landmark->set_visibility(landmark.visibility());
-    norm_landmark->set_presence(landmark.presence());
-  }
-}
-
-// Estimate object scale to use its inverse value as velocity scale for
-// RelativeVelocityFilter. If value will be too small (less than
-// `options_.min_allowed_object_scale`) smoothing will be disabled and
-// landmarks will be returned as is.
-// Object scale is calculated as average between bounding box width and height
-// with sides parallel to axis.
-float GetObjectScale(const LandmarkList& landmarks) {
-  const auto& lm_minmax_x = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.x() < b.x(); });
-  const float x_min = lm_minmax_x.first->x();
-  const float x_max = lm_minmax_x.second->x();
-
-  const auto& lm_minmax_y = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.y() < b.y(); });
-  const float y_min = lm_minmax_y.first->y();
-  const float y_max = lm_minmax_y.second->y();
-
-  const float object_width = x_max - x_min;
-  const float object_height = y_max - y_min;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const NormalizedRect& roi, const int image_width,
-                     const int image_height) {
-  const float object_width = roi.width() * image_width;
-  const float object_height = roi.height() * image_height;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const Rect& roi) {
-  return (roi.width() + roi.height()) / 2.0f;
-}
-
-// Abstract class for various landmarks filters.
-class LandmarksFilter {
- public:
-  virtual ~LandmarksFilter() = default;
-
-  virtual absl::Status Reset() { return absl::OkStatus(); }
-
-  virtual absl::Status Apply(const LandmarkList& in_landmarks,
-                             const absl::Duration& timestamp,
-                             const absl::optional<float> object_scale_opt,
-                             LandmarkList* out_landmarks) = 0;
-};
-
-// Returns landmarks as is without smoothing.
-class NoFilter : public LandmarksFilter {
- public:
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    *out_landmarks = in_landmarks;
-    return absl::OkStatus();
-  }
-};
-
-// Please check RelativeVelocityFilter documentation for details.
-class VelocityFilter : public LandmarksFilter {
- public:
-  VelocityFilter(int window_size, float velocity_scale,
-                 float min_allowed_object_scale, bool disable_value_scaling)
-      : window_size_(window_size),
-        velocity_scale_(velocity_scale),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    x_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    y_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    z_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-
-    return absl::OkStatus();
-  }
-
-  int window_size_;
-  float velocity_scale_;
-  float min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<RelativeVelocityFilter> x_filters_;
-  std::vector<RelativeVelocityFilter> y_filters_;
-  std::vector<RelativeVelocityFilter> z_filters_;
-};
-
-// Please check OneEuroFilter documentation for details.
-class OneEuroFilterImpl : public LandmarksFilter {
- public:
-  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
-                    double derivate_cutoff, float min_allowed_object_scale,
-                    bool disable_value_scaling)
-      : frequency_(frequency),
-        min_cutoff_(min_cutoff),
-        beta_(beta),
-        derivate_cutoff_(derivate_cutoff),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    for (int i = 0; i < n_landmarks; ++i) {
-      x_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      y_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      z_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-    }
-
-    return absl::OkStatus();
-  }
-
-  double frequency_;
-  double min_cutoff_;
-  double beta_;
-  double derivate_cutoff_;
-  double min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<OneEuroFilter> x_filters_;
-  std::vector<OneEuroFilter> y_filters_;
-  std::vector<OneEuroFilter> z_filters_;
-};
 
 }  // namespace
 
-// A calculator to smooth landmarks over time.
+class LandmarksSmoothingCalculatorImpl
-//
+    : public NodeImpl<LandmarksSmoothingCalculator> {
-// Inputs:
-//   NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
-//   IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
-//     Required to perform all computations in absolute coordinates to avoid any
-//     influence of normalized values.
-//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
-//     input landmarks) used to determine the object scale for some of the
-//     filters. If not provided - object scale will be calculated from
-//     landmarks.
-//
-// Outputs:
-//   NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
-//
-// Example config:
-//   node {
-//     calculator: "LandmarksSmoothingCalculator"
-//     input_stream: "NORM_LANDMARKS:pose_landmarks"
-//     input_stream: "IMAGE_SIZE:image_size"
-//     input_stream: "OBJECT_SCALE_ROI:roi"
-//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
-//     options: {
-//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
-//         velocity_filter: {
-//           window_size: 5
-//           velocity_scale: 10.0
-//         }
-//       }
-//     }
-//   }
-//
-class LandmarksSmoothingCalculator : public CalculatorBase {
  public:
-  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override {
-  absl::Status Open(CalculatorContext* cc) override;
+    ASSIGN_OR_RETURN(landmarks_filter_,
-  absl::Status Process(CalculatorContext* cc) override;
+                     InitializeLandmarksFilter(
+                         cc->Options<LandmarksSmoothingCalculatorOptions>()));
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if ((kInNormLandmarks(cc).IsConnected() &&
+         kInNormLandmarks(cc).IsEmpty()) ||
+        (kInLandmarks(cc).IsConnected() && kInLandmarks(cc).IsEmpty())) {
+      MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    if (kInNormLandmarks(cc).IsConnected()) {
+      const auto& in_norm_landmarks = kInNormLandmarks(cc).Get();
+
+      int image_width;
+      int image_height;
+      std::tie(image_width, image_height) = kImageSize(cc).Get();
+
+      absl::optional<float> object_scale;
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<NormalizedRect>();
+        object_scale = GetObjectScale(roi, image_width, image_height);
+      }
+
+      auto in_landmarks = absl::make_unique<LandmarkList>();
+      NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width,
+                                     image_height, *in_landmarks.get());
+
+      auto out_landmarks = absl::make_unique<LandmarkList>();
+      MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
+          *in_landmarks, timestamp, object_scale, *out_landmarks));
+
+      auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
+      LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
+                                     *out_norm_landmarks.get());
+
+      kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks));
+    } else {
+      const auto& in_landmarks = kInLandmarks(cc).Get();
+
+      absl::optional<float> object_scale;
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<Rect>();
+        object_scale = GetObjectScale(roi);
+      }
+
+      auto out_landmarks = absl::make_unique<LandmarkList>();
+      MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
+          in_landmarks, timestamp, object_scale, *out_landmarks));
+
+      kOutLandmarks(cc).Send(std::move(out_landmarks));
+    }
+
+    return absl::OkStatus();
+  }
 
  private:
   std::unique_ptr<LandmarksFilter> landmarks_filter_;
 };
-REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksSmoothingCalculatorImpl);
-
-absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-    cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
-    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
-    cc->Outputs()
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Set<NormalizedLandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<NormalizedRect>();
-    }
-  } else {
-    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
-    cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<Rect>();
-    }
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
-  cc->SetOffset(TimestampDiff(0));
-
-  // Pick landmarks filter.
-  const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
-  if (options.has_no_filter()) {
-    landmarks_filter_ = absl::make_unique<NoFilter>();
-  } else if (options.has_velocity_filter()) {
-    landmarks_filter_ = absl::make_unique<VelocityFilter>(
-        options.velocity_filter().window_size(),
-        options.velocity_filter().velocity_scale(),
-        options.velocity_filter().min_allowed_object_scale(),
-        options.velocity_filter().disable_value_scaling());
-  } else if (options.has_one_euro_filter()) {
-    landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
-        options.one_euro_filter().frequency(),
-        options.one_euro_filter().min_cutoff(),
-        options.one_euro_filter().beta(),
-        options.one_euro_filter().derivate_cutoff(),
-        options.one_euro_filter().min_allowed_object_scale(),
-        options.one_euro_filter().disable_value_scaling());
-  } else {
-    RET_CHECK_FAIL()
-        << "Landmarks filter is either not specified or not supported";
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
-  // Check that landmarks are not empty and reset the filter if so.
-  // Don't emit an empty packet for this timestamp.
-  if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
-       cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
-      (cc->Inputs().HasTag(kLandmarksTag) &&
-       cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
-    MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
-    return absl::OkStatus();
-  }
-
-  const auto& timestamp =
-      absl::Microseconds(cc->InputTimestamp().Microseconds());
-
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-    const auto& in_norm_landmarks =
-        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
-
-    int image_width;
-    int image_height;
-    std::tie(image_width, image_height) =
-        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
-
-    absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<NormalizedRect>();
-      object_scale = GetObjectScale(roi, image_width, image_height);
-    }
-
-    auto in_landmarks = absl::make_unique<LandmarkList>();
-    NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
-                                   in_landmarks.get());
-
-    auto out_landmarks = absl::make_unique<LandmarkList>();
-    MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        *in_landmarks, timestamp, object_scale, out_landmarks.get()));
-
-    auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
-    LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
-                                   out_norm_landmarks.get());
-
-    cc->Outputs()
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Add(out_norm_landmarks.release(), cc->InputTimestamp());
-  } else {
-    const auto& in_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
-
-    absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<Rect>();
-      object_scale = GetObjectScale(roi);
-    }
-
-    auto out_landmarks = absl::make_unique<LandmarkList>();
-    MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        in_landmarks, timestamp, object_scale, out_landmarks.get()));
-
-    cc->Outputs()
-        .Tag(kFilteredLandmarksTag)
-        .Add(out_landmarks.release(), cc->InputTimestamp());
-  }
-
-  return absl::OkStatus();
-}
 
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
+//     to smooth.
+//   LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
+//   IMAGE_SIZE (optional): A std::pair<int, int> represention of image width
+//     and height. Required to perform all computations in absolute coordinates
+//     when smoothing NORM_LANDMARKS to avoid any influence of normalized
+//     values.
+//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
+//     input landmarks) used to determine the object scale for some of the
+//     filters. If not provided - object scale will be calculated from
+//     landmarks.
+//
+// Outputs:
+//   NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
+//     landmarks.
+//   FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "LandmarksSmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:landmarks"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "NORM_FILTERED_LANDMARKS:landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class LandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
+      kInNormLandmarks{"NORM_LANDMARKS"};
+  static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarks{
+      "LANDMARKS"};
+  static constexpr Input<std::pair<int, int>>::Optional kImageSize{
+      "IMAGE_SIZE"};
+  static constexpr Input<OneOf<NormalizedRect, Rect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<mediapipe::NormalizedLandmarkList>::Optional
+      kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
+  static constexpr Output<mediapipe::LandmarkList>::Optional kOutLandmarks{
+      "FILTERED_LANDMARKS"};
+  MEDIAPIPE_NODE_INTERFACE(LandmarksSmoothingCalculator, kInNormLandmarks,
+                           kInLandmarks, kImageSize, kObjectScaleRoi,
+                           kOutNormLandmarks, kOutLandmarks);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK(kInNormLandmarks(cc).IsConnected() ^
+              kInLandmarks(cc).IsConnected())
+        << "One and only one of NORM_LANDMARKS and LANDMARKS input is allowed";
+
+    // TODO: Verify scale ROI is of the same type as landmarks
+    // that are being smoothed.
+
+    if (kInNormLandmarks(cc).IsConnected()) {
+      RET_CHECK(kImageSize(cc).IsConnected());
+      RET_CHECK(kOutNormLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutLandmarks(cc).IsConnected());
+    } else {
+      RET_CHECK(!kImageSize(cc).IsConnected());
+      RET_CHECK(kOutLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutNormLandmarks(cc).IsConnected());
+    }
+
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc

@@ -0,0 +1,375 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+
+#include <iostream>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+namespace {
+
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::OneEuroFilter;
+using ::mediapipe::Rect;
+using ::mediapipe::RelativeVelocityFilter;
+
+// Estimate object scale to use its inverse value as velocity scale for
+// RelativeVelocityFilter. If value will be too small (less than
+// `options_.min_allowed_object_scale`) smoothing will be disabled and
+// landmarks will be returned as is.
+// Object scale is calculated as average between bounding box width and height
+// with sides parallel to axis.
+float GetObjectScale(const LandmarkList& landmarks) {
+  const auto& lm_minmax_x = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.x() < b.x(); });
+  const float x_min = lm_minmax_x.first->x();
+  const float x_max = lm_minmax_x.second->x();
+
+  const auto& lm_minmax_y = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.y() < b.y(); });
+  const float y_min = lm_minmax_y.first->y();
+  const float y_max = lm_minmax_y.second->y();
+
+  const float object_width = x_max - x_min;
+  const float object_height = y_max - y_min;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+// Returns landmarks as is without smoothing.
+class NoFilter : public LandmarksFilter {
+ public:
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    out_landmarks = in_landmarks;
+    return absl::OkStatus();
+  }
+};
+
+// Please check RelativeVelocityFilter documentation for details.
+class VelocityFilter : public LandmarksFilter {
+ public:
+  VelocityFilter(int window_size, float velocity_scale,
+                 float min_allowed_object_scale, bool disable_value_scaling)
+      : window_size_(window_size),
+        velocity_scale_(velocity_scale),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    x_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    y_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    z_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+
+    return absl::OkStatus();
+  }
+
+  int window_size_;
+  float velocity_scale_;
+  float min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<RelativeVelocityFilter> x_filters_;
+  std::vector<RelativeVelocityFilter> y_filters_;
+  std::vector<RelativeVelocityFilter> z_filters_;
+};
+
+// Please check OneEuroFilter documentation for details.
+class OneEuroFilterImpl : public LandmarksFilter {
+ public:
+  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
+                    double derivate_cutoff, float min_allowed_object_scale,
+                    bool disable_value_scaling)
+      : frequency_(frequency),
+        min_cutoff_(min_cutoff),
+        beta_(beta),
+        derivate_cutoff_(derivate_cutoff),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    for (int i = 0; i < n_landmarks; ++i) {
+      x_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      y_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      z_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+    }
+
+    return absl::OkStatus();
+  }
+
+  double frequency_;
+  double min_cutoff_;
+  double beta_;
+  double derivate_cutoff_;
+  double min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<OneEuroFilter> x_filters_;
+  std::vector<OneEuroFilter> y_filters_;
+  std::vector<OneEuroFilter> z_filters_;
+};
+
+}  // namespace
+
+void NormalizedLandmarksToLandmarks(
+    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+    const int image_height, LandmarkList& landmarks) {
+  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
+    const auto& norm_landmark = norm_landmarks.landmark(i);
+
+    auto* landmark = landmarks.add_landmark();
+    landmark->set_x(norm_landmark.x() * image_width);
+    landmark->set_y(norm_landmark.y() * image_height);
+    // Scale Z the same way as X (using image width).
+    landmark->set_z(norm_landmark.z() * image_width);
+
+    if (norm_landmark.has_visibility()) {
+      landmark->set_visibility(norm_landmark.visibility());
+    } else {
+      landmark->clear_visibility();
+    }
+
+    if (norm_landmark.has_presence()) {
+      landmark->set_presence(norm_landmark.presence());
+    } else {
+      landmark->clear_presence();
+    }
+  }
+}
+
+void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
+                                    const int image_width,
+                                    const int image_height,
+                                    NormalizedLandmarkList& norm_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+
+    auto* norm_landmark = norm_landmarks.add_landmark();
+    norm_landmark->set_x(landmark.x() / image_width);
+    norm_landmark->set_y(landmark.y() / image_height);
+    // Scale Z the same way as X (using image width).
+    norm_landmark->set_z(landmark.z() / image_width);
+
+    if (landmark.has_visibility()) {
+      norm_landmark->set_visibility(landmark.visibility());
+    } else {
+      norm_landmark->clear_visibility();
+    }
+
+    if (landmark.has_presence()) {
+      norm_landmark->set_presence(landmark.presence());
+    } else {
+      norm_landmark->clear_presence();
+    }
+  }
+}
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height) {
+  const float object_width = roi.width() * image_width;
+  const float object_height = roi.height() * image_height;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+float GetObjectScale(const Rect& roi) {
+  return (roi.width() + roi.height()) / 2.0f;
+}
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const LandmarksSmoothingCalculatorOptions& options) {
+  if (options.has_no_filter()) {
+    return absl::make_unique<NoFilter>();
+  } else if (options.has_velocity_filter()) {
+    return absl::make_unique<VelocityFilter>(
+        options.velocity_filter().window_size(),
+        options.velocity_filter().velocity_scale(),
+        options.velocity_filter().min_allowed_object_scale(),
+        options.velocity_filter().disable_value_scaling());
+  } else if (options.has_one_euro_filter()) {
+    return absl::make_unique<OneEuroFilterImpl>(
+        options.one_euro_filter().frequency(),
+        options.one_euro_filter().min_cutoff(),
+        options.one_euro_filter().beta(),
+        options.one_euro_filter().derivate_cutoff(),
+        options.one_euro_filter().min_allowed_object_scale(),
+        options.one_euro_filter().disable_value_scaling());
+  } else {
+    RET_CHECK_FAIL()
+        << "Landmarks filter is either not specified or not supported";
+  }
+}
+
+absl::StatusOr<LandmarksFilter*> MultiLandmarkFilters::GetOrCreate(
+    const int64_t tracking_id,
+    const mediapipe::LandmarksSmoothingCalculatorOptions& options) {
+  const auto it = filters_.find(tracking_id);
+  if (it != filters_.end()) {
+    return it->second.get();
+  }
+
+  ASSIGN_OR_RETURN(auto landmarks_filter, InitializeLandmarksFilter(options));
+  filters_[tracking_id] = std::move(landmarks_filter);
+  return filters_[tracking_id].get();
+}
+
+void MultiLandmarkFilters::ClearUnused(
+    const std::vector<int64_t>& tracking_ids) {
+  std::vector<int64_t> unused_tracking_ids;
+  for (const auto& it : filters_) {
+    bool unused = true;
+    for (int64_t tracking_id : tracking_ids) {
+      if (tracking_id == it.first) unused = false;
+    }
+    if (unused) unused_tracking_ids.push_back(it.first);
+  }
+
+  for (int64_t tracking_id : unused_tracking_ids) {
+    filters_.erase(tracking_id);
+  }
+}
+
+void MultiLandmarkFilters::Clear() { filters_.clear(); }
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h

@@ -0,0 +1,77 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+void NormalizedLandmarksToLandmarks(
+    const mediapipe::NormalizedLandmarkList& norm_landmarks,
+    const int image_width, const int image_height,
+    mediapipe::LandmarkList& landmarks);
+
+void LandmarksToNormalizedLandmarks(
+    const mediapipe::LandmarkList& landmarks, const int image_width,
+    const int image_height, mediapipe::NormalizedLandmarkList& norm_landmarks);
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height);
+
+float GetObjectScale(const Rect& roi);
+
+// Abstract class for various landmarks filters.
+class LandmarksFilter {
+ public:
+  virtual ~LandmarksFilter() = default;
+
+  virtual absl::Status Reset() { return absl::OkStatus(); }
+
+  virtual absl::Status Apply(const mediapipe::LandmarkList& in_landmarks,
+                             const absl::Duration& timestamp,
+                             const absl::optional<float> object_scale_opt,
+                             mediapipe::LandmarkList& out_landmarks) = 0;
+};
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const mediapipe::LandmarksSmoothingCalculatorOptions& options);
+
+class MultiLandmarkFilters {
+ public:
+  virtual ~MultiLandmarkFilters() = default;
+
+  virtual absl::StatusOr<LandmarksFilter*> GetOrCreate(
+      const int64_t tracking_id,
+      const mediapipe::LandmarksSmoothingCalculatorOptions& options);
+
+  virtual void ClearUnused(const std::vector<int64_t>& tracking_ids);
+
+  virtual void Clear();
+
+ private:
+  std::map<int64_t, std::unique_ptr<LandmarksFilter>> filters_;
+};
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_

mediapipe/calculators/util/landmarks_smoothing_calculator_utils_test.cc

@@ -0,0 +1,118 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+namespace {
+
+TEST(LandmarksSmoothingCalculatorUtilsTest, NormalizedLandmarksToLandmarks) {
+  NormalizedLandmarkList norm_landmarks;
+  NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
+  norm_landmark->set_x(0.1);
+  norm_landmark->set_y(0.2);
+  norm_landmark->set_z(0.3);
+  norm_landmark->set_visibility(0.4);
+  norm_landmark->set_presence(0.5);
+
+  LandmarkList landmarks;
+  NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, landmarks);
+
+  EXPECT_EQ(landmarks.landmark_size(), 1);
+  Landmark landmark = landmarks.landmark(0);
+  EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
+  EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
+  EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
+  EXPECT_NEAR(landmark.visibility(), 0.4, 1e-6);
+  EXPECT_NEAR(landmark.presence(), 0.5, 1e-6);
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest,
+     NormalizedLandmarksToLandmarks_EmptyVisibilityAndPresence) {
+  NormalizedLandmarkList norm_landmarks;
+  NormalizedLandmark* norm_landmark = norm_landmarks.add_landmark();
+  norm_landmark->set_x(0.1);
+  norm_landmark->set_y(0.2);
+  norm_landmark->set_z(0.3);
+  norm_landmark->clear_visibility();
+  norm_landmark->clear_presence();
+
+  LandmarkList landmarks;
+  NormalizedLandmarksToLandmarks(norm_landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, landmarks);
+
+  EXPECT_EQ(landmarks.landmark_size(), 1);
+  Landmark landmark = landmarks.landmark(0);
+  EXPECT_NEAR(landmark.x(), 1.0, 1e-6);
+  EXPECT_NEAR(landmark.y(), 2.0, 1e-6);
+  EXPECT_NEAR(landmark.z(), 3.0, 1e-6);
+  EXPECT_FALSE(landmark.has_visibility());
+  EXPECT_FALSE(landmark.has_presence());
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest, LandmarksToNormalizedLandmarks) {
+  LandmarkList landmarks;
+  Landmark* landmark = landmarks.add_landmark();
+  landmark->set_x(1.0);
+  landmark->set_y(2.0);
+  landmark->set_z(3.0);
+  landmark->set_visibility(0.4);
+  landmark->set_presence(0.5);
+
+  NormalizedLandmarkList norm_landmarks;
+  LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, norm_landmarks);
+
+  EXPECT_EQ(norm_landmarks.landmark_size(), 1);
+  NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
+  EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
+  EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
+  EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
+  EXPECT_NEAR(norm_landmark.visibility(), 0.4, 1e-6);
+  EXPECT_NEAR(norm_landmark.presence(), 0.5, 1e-6);
+}
+
+TEST(LandmarksSmoothingCalculatorUtilsTest,
+     LandmarksToNormalizedLandmarks_EmptyVisibilityAndPresence) {
+  LandmarkList landmarks;
+  Landmark* landmark = landmarks.add_landmark();
+  landmark->set_x(1.0);
+  landmark->set_y(2.0);
+  landmark->set_z(3.0);
+  landmark->clear_visibility();
+  landmark->clear_presence();
+
+  NormalizedLandmarkList norm_landmarks;
+  LandmarksToNormalizedLandmarks(landmarks, /*image_width=*/10,
+                                 /*image_height=*/10, norm_landmarks);
+
+  EXPECT_EQ(norm_landmarks.landmark_size(), 1);
+  NormalizedLandmark norm_landmark = norm_landmarks.landmark(0);
+  EXPECT_NEAR(norm_landmark.x(), 0.1, 1e-6);
+  EXPECT_NEAR(norm_landmark.y(), 0.2, 1e-6);
+  EXPECT_NEAR(norm_landmark.z(), 0.3, 1e-6);
+  EXPECT_FALSE(norm_landmark.has_visibility());
+  EXPECT_FALSE(norm_landmark.has_presence());
+}
+
+}  // namespace
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -322,27 +322,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
           options_.presence_threshold(), options_.connection_color(), thickness,
           /*normalized=*/false, render_data.get());
     }
-    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    if (options_.render_landmarks()) {
-      const Landmark& landmark = landmarks.landmark(i);
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        const Landmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibleAndPresent<Landmark>(
+        if (!IsLandmarkVisibleAndPresent<Landmark>(
-              landmark, options_.utilize_visibility(),
+                landmark, options_.utilize_visibility(),
-              options_.visibility_threshold(), options_.utilize_presence(),
+                options_.visibility_threshold(), options_.utilize_presence(),
-              options_.presence_threshold())) {
+                options_.presence_threshold())) {
-        continue;
+          continue;
-      }
+        }
 
-      auto* landmark_data_render = AddPointRenderData(
+        auto* landmark_data_render = AddPointRenderData(
-          options_.landmark_color(), thickness, render_data.get());
+            options_.landmark_color(), thickness, render_data.get());
-      if (visualize_depth) {
+        if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
+          SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
-                               options_.min_depth_circle_thickness(),
+                                 landmark_data_render,
-                               options_.max_depth_circle_thickness());
+                                 options_.min_depth_circle_thickness(),
+                                 options_.max_depth_circle_thickness());
+        }
+        auto* landmark_data = landmark_data_render->mutable_point();
+        landmark_data->set_normalized(false);
+        landmark_data->set_x(landmark.x());
+        landmark_data->set_y(landmark.y());
       }
-      auto* landmark_data = landmark_data_render->mutable_point();
-      landmark_data->set_normalized(false);
-      landmark_data->set_x(landmark.x());
-      landmark_data->set_y(landmark.y());
     }
   }
 
@@ -368,27 +371,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
           options_.presence_threshold(), options_.connection_color(), thickness,
           /*normalized=*/true, render_data.get());
     }
-    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    if (options_.render_landmarks()) {
-      const NormalizedLandmark& landmark = landmarks.landmark(i);
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+        const NormalizedLandmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
+        if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
-              landmark, options_.utilize_visibility(),
+                landmark, options_.utilize_visibility(),
-              options_.visibility_threshold(), options_.utilize_presence(),
+                options_.visibility_threshold(), options_.utilize_presence(),
-              options_.presence_threshold())) {
+                options_.presence_threshold())) {
-        continue;
+          continue;
-      }
+        }
 
-      auto* landmark_data_render = AddPointRenderData(
+        auto* landmark_data_render = AddPointRenderData(
-          options_.landmark_color(), thickness, render_data.get());
+            options_.landmark_color(), thickness, render_data.get());
-      if (visualize_depth) {
+        if (visualize_depth) {
-        SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
+          SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
-                               options_.min_depth_circle_thickness(),
+                                 landmark_data_render,
-                               options_.max_depth_circle_thickness());
+                                 options_.min_depth_circle_thickness(),
+                                 options_.max_depth_circle_thickness());
+        }
+        auto* landmark_data = landmark_data_render->mutable_point();
+        landmark_data->set_normalized(true);
+        landmark_data->set_x(landmark.x());
+        landmark_data->set_y(landmark.y());
       }
-      auto* landmark_data = landmark_data_render->mutable_point();
-      landmark_data->set_normalized(true);
-      landmark_data->set_x(landmark.x());
-      landmark_data->set_y(landmark.y());
     }
   }
 

mediapipe/calculators/util/landmarks_to_render_data_calculator.proto

@@ -32,6 +32,10 @@ message LandmarksToRenderDataCalculatorOptions {
 
   // Color of the landmarks.
   optional Color landmark_color = 2;
+
+  // Whether to render landmarks as points.
+  optional bool render_landmarks = 14 [default = true];
+
   // Color of the connections.
   optional Color connection_color = 3;
 

mediapipe/calculators/util/multi_landmarks_smoothing_calculator.cc

@@ -0,0 +1,113 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
+using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
+using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
+using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
+
+}  // namespace
+
+class MultiLandmarksSmoothingCalculatorImpl
+    : public NodeImpl<MultiLandmarksSmoothingCalculator> {
+ public:
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if (kInNormLandmarks(cc).IsEmpty()) {
+      multi_filters_.Clear();
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    const auto& tracking_ids = kTrackingIds(cc).Get();
+    multi_filters_.ClearUnused(tracking_ids);
+
+    const auto& in_norm_landmarks_vec = kInNormLandmarks(cc).Get();
+    RET_CHECK_EQ(in_norm_landmarks_vec.size(), tracking_ids.size());
+
+    int image_width;
+    int image_height;
+    std::tie(image_width, image_height) = kImageSize(cc).Get();
+
+    std::optional<std::vector<NormalizedRect>> object_scale_roi_vec;
+    if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+      object_scale_roi_vec = kObjectScaleRoi(cc).Get();
+      RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
+    }
+
+    std::vector<NormalizedLandmarkList> out_norm_landmarks_vec;
+    for (int i = 0; i < tracking_ids.size(); ++i) {
+      LandmarkList in_landmarks;
+      NormalizedLandmarksToLandmarks(in_norm_landmarks_vec[i], image_width,
+                                     image_height, in_landmarks);
+
+      std::optional<float> object_scale;
+      if (object_scale_roi_vec) {
+        object_scale = GetObjectScale(object_scale_roi_vec.value()[i],
+                                      image_width, image_height);
+      }
+
+      ASSIGN_OR_RETURN(auto* landmarks_filter,
+                       multi_filters_.GetOrCreate(
+                           tracking_ids[i],
+                           cc->Options<LandmarksSmoothingCalculatorOptions>()));
+
+      LandmarkList out_landmarks;
+      MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
+                                                 object_scale, out_landmarks));
+
+      NormalizedLandmarkList out_norm_landmarks;
+      LandmarksToNormalizedLandmarks(out_landmarks, image_width, image_height,
+                                     out_norm_landmarks);
+
+      out_norm_landmarks_vec.push_back(std::move(out_norm_landmarks));
+    }
+
+    kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks_vec));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  MultiLandmarkFilters multi_filters_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(MultiLandmarksSmoothingCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/multi_landmarks_smoothing_calculator.h

@@ -0,0 +1,81 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   NORM_LANDMARKS: A std::vector<NormalizedLandmarkList> of landmarks you want
+//     to smooth.
+//   TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
+//     landmarks over time. When new ID arrives - calculator will initialize new
+//     filter. When tracking ID is no longer provided - calculator will forget
+//     smoothing state.
+//   IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
+//     Required to perform all computations in absolute coordinates to avoid any
+//     influence of normalized values.
+//   OBJECT_SCALE_ROI (optional): A std::vector<NormRect> used to determine the
+//     object scale for some of the filters. If not provided - object scale will
+//     be calculated from landmarks.
+//
+// Outputs:
+//   NORM_FILTERED_LANDMARKS: A std::vector<NormalizedLandmarkList> of smoothed
+//     landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "MultiLandmarksSmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:pose_landmarks"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class MultiLandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<std::vector<mediapipe::NormalizedLandmarkList>>
+      kInNormLandmarks{"NORM_LANDMARKS"};
+  static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
+  static constexpr Input<std::pair<int, int>> kImageSize{"IMAGE_SIZE"};
+  static constexpr Input<std::vector<NormalizedRect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<std::vector<mediapipe::NormalizedLandmarkList>>
+      kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(MultiLandmarksSmoothingCalculator, kInNormLandmarks,
+                           kTrackingIds, kImageSize, kObjectScaleRoi,
+                           kOutNormLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_MULTI_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.cc

@@ -0,0 +1,100 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <vector>
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
+
+namespace mediapipe {
+namespace api2 {
+
+namespace {
+
+using ::mediapipe::Rect;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
+using ::mediapipe::landmarks_smoothing::MultiLandmarkFilters;
+
+}  // namespace
+
+class MultiWorldLandmarksSmoothingCalculatorImpl
+    : public NodeImpl<MultiWorldLandmarksSmoothingCalculator> {
+ public:
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty and reset the filter if so.
+    // Don't emit an empty packet for this timestamp.
+    if (kInLandmarks(cc).IsEmpty()) {
+      multi_filters_.Clear();
+      return absl::OkStatus();
+    }
+
+    const auto& timestamp =
+        absl::Microseconds(cc->InputTimestamp().Microseconds());
+
+    const auto& tracking_ids = kTrackingIds(cc).Get();
+    multi_filters_.ClearUnused(tracking_ids);
+
+    const auto& in_landmarks_vec = kInLandmarks(cc).Get();
+    RET_CHECK_EQ(in_landmarks_vec.size(), tracking_ids.size());
+
+    std::optional<std::vector<Rect>> object_scale_roi_vec;
+    if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
+      object_scale_roi_vec = kObjectScaleRoi(cc).Get();
+      RET_CHECK_EQ(object_scale_roi_vec.value().size(), tracking_ids.size());
+    }
+
+    std::vector<LandmarkList> out_landmarks_vec;
+    for (int i = 0; i < tracking_ids.size(); ++i) {
+      const auto& in_landmarks = in_landmarks_vec[i];
+
+      std::optional<float> object_scale;
+      if (object_scale_roi_vec) {
+        object_scale = GetObjectScale(object_scale_roi_vec.value()[i]);
+      }
+
+      ASSIGN_OR_RETURN(auto* landmarks_filter,
+                       multi_filters_.GetOrCreate(
+                           tracking_ids[i],
+                           cc->Options<LandmarksSmoothingCalculatorOptions>()));
+
+      LandmarkList out_landmarks;
+      MP_RETURN_IF_ERROR(landmarks_filter->Apply(in_landmarks, timestamp,
+                                                 object_scale, out_landmarks));
+
+      out_landmarks_vec.push_back(std::move(out_landmarks));
+    }
+
+    kOutLandmarks(cc).Send(std::move(out_landmarks_vec));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  MultiLandmarkFilters multi_filters_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(MultiWorldLandmarksSmoothingCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/multi_world_landmarks_smoothing_calculator.h

@@ -0,0 +1,74 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   LANDMARKS: A std::vector<LandmarkList> of landmarks you want to
+//     smooth.
+//   TRACKING_IDS: A std<int64_t> vector of tracking IDs used to associate
+//     landmarks over time. When new ID arrives - calculator will initialize new
+//     filter. When tracking ID is no longer provided - calculator will forget
+//     smoothing state.
+//   OBJECT_SCALE_ROI (optional): A std::vector<Rect> used to determine the
+//     object scale for some of the filters. If not provided - object scale will
+//     be calculated from landmarks.
+//
+// Outputs:
+//   FILTERED_LANDMARKS: A std::vector<LandmarkList> of smoothed landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "MultiWorldLandmarksSmoothingCalculator"
+//     input_stream: "LANDMARKS:landmarks"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "FILTERED_LANDMARKS:landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class MultiWorldLandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<std::vector<mediapipe::LandmarkList>> kInLandmarks{
+      "LANDMARKS"};
+  static constexpr Input<std::vector<int64_t>> kTrackingIds{"TRACKING_IDS"};
+  static constexpr Input<std::vector<Rect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<std::vector<mediapipe::LandmarkList>> kOutLandmarks{
+      "FILTERED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(MultiWorldLandmarksSmoothingCalculator, kInLandmarks,
+                           kTrackingIds, kObjectScaleRoi, kOutLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_MULTI_WORLD_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc

@@ -124,7 +124,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     int center_row = out_lms.landmark(lm_index).y() * hm_height;
     // Point is outside of the image let's keep it intact.
     if (center_col < 0 || center_col >= hm_width || center_row < 0 ||
-        center_col >= hm_height) {
+        center_row >= hm_height) {
       continue;
     }
 

mediapipe/calculators/video/BUILD

@@ -130,7 +130,6 @@ cc_library(
         "//mediapipe/framework/formats:video_stream_header",
         "//mediapipe/framework/port:opencv_imgproc",
         "//mediapipe/framework/port:opencv_video",
-        "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:status_util",
     ],
@@ -341,7 +340,6 @@ cc_test(
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:test_util",
-        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -367,7 +365,6 @@ cc_test(
         "//mediapipe/framework/port:opencv_video",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:test_util",
-        "@com_google_absl//absl/flags:flag",
     ],
 )
 
@@ -451,7 +448,6 @@ cc_test(
         "//mediapipe/framework/tool:test_util",
         "//mediapipe/util/tracking:box_tracker_cc_proto",
         "//mediapipe/util/tracking:tracking_cc_proto",
-        "@com_google_absl//absl/flags:flag",
     ],
 )
 

mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

mediapipe/examples/ios/facedetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectioncpu",

mediapipe/examples/ios/facedetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facedetectiongpu",

mediapipe/examples/ios/faceeffect/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "faceeffect",

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "facemeshgpu",

mediapipe/examples/ios/handdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handdetectiongpu",

mediapipe/examples/ios/handtrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "handtrackinggpu",

mediapipe/examples/ios/helloworld/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "helloworld",

mediapipe/examples/ios/holistictrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "holistictrackinggpu",

mediapipe/examples/ios/iristrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "iristrackinggpu",

mediapipe/examples/ios/objectdetectioncpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectioncpu",

mediapipe/examples/ios/objectdetectiongpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiongpu",

mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "objectdetectiontrackinggpu",

mediapipe/examples/ios/posetrackinggpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "posetrackinggpu",

mediapipe/examples/ios/selfiesegmentationgpu/BUILD

@@ -24,7 +24,7 @@ load(
 
 licenses(["notice"])
 
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
 
 alias(
     name = "selfiesegmentationgpu",

mediapipe/framework/BUILD

@@ -44,6 +44,9 @@ bzl_library(
         "encode_binary_proto.bzl",
     ],
     visibility = ["//visibility:public"],
+    deps = [
+        "@bazel_skylib//lib:paths",
+    ],
 )
 
 alias(
@@ -1099,6 +1102,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
     ],
+    alwayslink = True,  # Defines TestServiceCalculator
 )
 
 cc_library(
@@ -1354,6 +1358,23 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "calculator_graph_summary_packet_test",
+    srcs = ["calculator_graph_summary_packet_test.cc"],
+    deps = [
+        ":calculator_framework",
+        ":packet",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:packet",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+        "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/status",
+    ],
+)
+
 cc_test(
     name = "calculator_runner_test",
     size = "medium",
@@ -1405,6 +1426,7 @@ cc_test(
         "calculator_graph_test.cc",
     ],
     linkstatic = 1,
+    tags = ["not_run:arm"],
     visibility = ["//visibility:public"],
     deps = [
         ":calculator_framework",

mediapipe/framework/api2/builder.h

@@ -32,7 +32,7 @@ template <class T>
 struct dependent_false : std::false_type {};
 
 template <typename T>
-T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, int index) {
+T& GetWithAutoGrow(std::vector<std::unique_ptr<T>>* vecp, size_t index) {
   auto& vec = *vecp;
   if (vec.size() <= index) {
     vec.resize(index + 1);

mediapipe/framework/api2/node.h

@@ -64,58 +64,13 @@ class CalculatorBaseFactoryFor<
 namespace api2 {
 namespace internal {
 
-// Defining a member of this type causes P to be ODR-used, which forces its
+MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(
-// instantiation if it's a static member of a template.
+    NodeRegistrator, mediapipe::CalculatorBaseRegistry, T::kCalculatorName,
-// Previously we depended on the pointer's value to determine whether the size
+    absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>)
-// of a character array is 0 or 1, forcing it to be instantiated so the
-// compiler can determine the object's layout. But using it as a template
-// argument is more compact.
-template <auto* P>
-struct ForceStaticInstantiation {
-#ifdef _MSC_VER
-  // Just having it as the template argument does not count as a use for
-  // MSVC.
-  static constexpr bool Use() { return P != nullptr; }
-  char force_static[Use()];
-#endif  // _MSC_VER
-};
 
-// Helper template for forcing the definition of a static registration token.
+MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator,
-template <typename T>
+                                      mediapipe::SubgraphRegistry,
-struct NodeRegistrationStatic {
+                                      T::kCalculatorName, absl::make_unique<T>)
-  static NoDestructor<mediapipe::RegistrationToken> registration;
-
-  static mediapipe::RegistrationToken Make() {
-    return mediapipe::CalculatorBaseRegistry::Register(
-        T::kCalculatorName,
-        absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<T>>,
-        __FILE__, __LINE__);
-  }
-
-  using RequireStatics = ForceStaticInstantiation<&registration>;
-};
-
-// Static members of template classes can be defined in the header.
-template <typename T>
-NoDestructor<mediapipe::RegistrationToken>
-    NodeRegistrationStatic<T>::registration(NodeRegistrationStatic<T>::Make());
-
-template <typename T>
-struct SubgraphRegistrationImpl {
-  static NoDestructor<mediapipe::RegistrationToken> registration;
-
-  static mediapipe::RegistrationToken Make() {
-    return mediapipe::SubgraphRegistry::Register(
-        T::kCalculatorName, absl::make_unique<T>, __FILE__, __LINE__);
-  }
-
-  using RequireStatics = ForceStaticInstantiation<&registration>;
-};
-
-template <typename T>
-NoDestructor<mediapipe::RegistrationToken>
-    SubgraphRegistrationImpl<T>::registration(
-        SubgraphRegistrationImpl<T>::Make());
 
 }  // namespace internal
 
@@ -128,14 +83,7 @@ template <class Impl = void>
 class RegisteredNode;
 
 template <class Impl>
-class RegisteredNode : public Node {
+class RegisteredNode : public Node, private internal::NodeRegistrator<Impl> {};
- private:
-  // The member below triggers instantiation of the registration static.
-  // Note that the constructor of calculator subclasses is only invoked through
-  // the registration token, and so we cannot simply use the static in the
-  // constructor.
-  typename internal::NodeRegistrationStatic<Impl>::RequireStatics register_;
-};
 
 // No-op version for backwards compatibility.
 template <>
@@ -217,31 +165,27 @@ class NodeImpl : public RegisteredNode<Impl>, public Intf {
 // TODO: verify that the subgraph config fully implements the
 // declared interface.
 template <class Intf, class Impl>
-class SubgraphImpl : public Subgraph, public Intf {
+class SubgraphImpl : public Subgraph,
- private:
+                     public Intf,
-  typename internal::SubgraphRegistrationImpl<Impl>::RequireStatics register_;
+                     private internal::SubgraphRegistrator<Impl> {};
-};
 
 // This macro is used to register a calculator that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                                   \
+#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl)                       \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>                \
+  MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(                  \
-  REGISTRY_STATIC_VAR(calculator_registration,                                \
+      mediapipe::CalculatorBaseRegistry, calculator_registration, \
-                      __LINE__)(mediapipe::CalculatorBaseRegistry::Register(  \
+      Impl::kCalculatorName,                                      \
-      Impl::kCalculatorName,                                                  \
+      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>)
-      absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>, \
-      __FILE__, __LINE__))
 
 // This macro is used to register a non-split-contract calculator. Deprecated.
 #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
 
 // This macro is used to define a subgraph that does not use automatic
 // registration. Deprecated.
-#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)                        \
+#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl)           \
-  static mediapipe::NoDestructor<mediapipe::RegistrationToken>         \
+  MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(          \
-  REGISTRY_STATIC_VAR(subgraph_registration,                           \
+      mediapipe::SubgraphRegistry, subgraph_registration, \
-                      __LINE__)(mediapipe::SubgraphRegistry::Register( \
+      Impl::kCalculatorName, absl::make_unique<Impl>)
-      Impl::kCalculatorName, absl::make_unique<Impl>, __FILE__, __LINE__))
 
 }  // namespace api2
 }  // namespace mediapipe

mediapipe/framework/api2/packet.h

@@ -165,7 +165,7 @@ template <class V, class... U>
 struct IsCompatibleType<V, OneOf<U...>>
     : std::integral_constant<bool, (std::is_same_v<V, U> || ...)> {};
 
-};  // namespace internal
+}  // namespace internal
 
 template <typename T>
 inline Packet<T> PacketBase::As() const {
@@ -259,19 +259,19 @@ struct First {
 
 template <class T>
 struct AddStatus {
-  using type = StatusOr<T>;
+  using type = absl::StatusOr<T>;
 };
 template <class T>
-struct AddStatus<StatusOr<T>> {
+struct AddStatus<absl::StatusOr<T>> {
-  using type = StatusOr<T>;
+  using type = absl::StatusOr<T>;
 };
 template <>
-struct AddStatus<Status> {
+struct AddStatus<absl::Status> {
-  using type = Status;
+  using type = absl::Status;
 };
 template <>
 struct AddStatus<void> {
-  using type = Status;
+  using type = absl::Status;
 };
 
 template <class R, class F, class... A>
@@ -282,7 +282,7 @@ struct CallAndAddStatusImpl {
 };
 template <class F, class... A>
 struct CallAndAddStatusImpl<void, F, A...> {
-  Status operator()(const F& f, A&&... a) {
+  absl::Status operator()(const F& f, A&&... a) {
     f(std::forward<A>(a)...);
     return {};
   }

mediapipe/framework/calculator_base_test.cc

@@ -183,8 +183,7 @@ TEST(CalculatorTest, CreateByNameWhitelisted) {
   CalculatorBaseRegistry::Register(
       "::mediapipe::test_ns::whitelisted_ns::DeadCalculator",
       absl::make_unique<internal::CalculatorBaseFactoryFor<
-          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>,
+          mediapipe::test_ns::whitelisted_ns::DeadCalculator>>);
-      __FILE__, __LINE__);
 
   // A whitelisted calculator can be found in its own namespace.
   MP_EXPECT_OK(CalculatorBaseRegistry::CreateByNameInNamespace(  //

mediapipe/framework/calculator_context.h

@@ -109,9 +109,20 @@ class CalculatorContext {
   // use OutputStream::SetOffset() directly.
   void SetOffset(TimestampDiff offset);
 
-  // Returns the status of the graph run.
+  // DEPRECATED: This was intended to get graph run status during
+  // `CalculatorBase::Close` call. However, `Close` can run simultaneously with
+  // other calculators `CalculatorBase::Process`, hence the actual graph
+  // status may change any time and returned graph status here does not
+  // necessarily reflect the actual graph status.
   //
-  // NOTE: This method should only be called during CalculatorBase::Close().
+  // As an alternative, instead of checking graph status in `Close` and doing
+  // work for "done" state, you can enable timestamp bound processing for your
+  // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger
+  // `Process` on timestamp bound updates and handle "done" state there.
+  // Check examples in:
+  // mediapipe/framework/calculator_graph_summary_packet_test.cc.
+  //
+  ABSL_DEPRECATED("Does not reflect the actual graph status.")
   absl::Status GraphStatus() const { return graph_status_; }
 
   ProfilingContext* GetProfilingContext() const {

mediapipe/framework/calculator_graph.cc

@@ -839,6 +839,13 @@ absl::Status CalculatorGraph::PrepareForRun(
 }
 
 absl::Status CalculatorGraph::WaitUntilIdle() {
+  if (has_sources_) {
+    LOG_FIRST_N(WARNING, 1)
+        << "WaitUntilIdle called on a graph with source nodes, which "
+           "is not fully supported at the moment. Source nodes: "
+        << ListSourceNodes();
+  }
+
   MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle());
   VLOG(2) << "Scheduler idle.";
   absl::Status status = absl::OkStatus();
@@ -1368,6 +1375,16 @@ const OutputStreamManager* CalculatorGraph::FindOutputStreamManager(
               .get()[validated_graph_->OutputStreamIndex(name)];
 }
 
+std::string CalculatorGraph::ListSourceNodes() const {
+  std::vector<std::string> sources;
+  for (auto& node : nodes_) {
+    if (node->IsSource()) {
+      sources.push_back(node->DebugName());
+    }
+  }
+  return absl::StrJoin(sources, ", ");
+}
+
 namespace {
 void PrintTimingToInfo(const std::string& label, int64_t timer_value) {
   const int64_t total_seconds = timer_value / 1000000ll;

mediapipe/framework/calculator_graph.h

@@ -229,8 +229,11 @@ class CalculatorGraph {
   // Wait until the running graph is in the idle mode, which is when nothing can
   // be scheduled and nothing is running in the worker threads. This function
   // can be called only after StartRun().
+  //
   // NOTE: The graph must not have any source nodes because source nodes prevent
   // the running graph from becoming idle until the source nodes are done.
+  // Currently, `WaitUntilIdle` cannot be used reliably on graphs with any
+  // source nodes.
   absl::Status WaitUntilIdle();
 
   // Wait until a packet is emitted on one of the observed output streams.
@@ -594,6 +597,9 @@ class CalculatorGraph {
   // status before taking any action.
   void UpdateThrottledNodes(InputStreamManager* stream, bool* stream_was_full);
 
+  // Returns a comma-separated list of source nodes.
+  std::string ListSourceNodes() const;
+
 #if !MEDIAPIPE_DISABLE_GPU
   // Owns the legacy GpuSharedData if we need to create one for backwards
   // compatibility.

mediapipe/framework/calculator_graph_summary_packet_test.cc

@@ -0,0 +1,430 @@
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+using ::mediapipe::api2::Input;
+using ::mediapipe::api2::Node;
+using ::mediapipe::api2::Output;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsEmpty;
+using ::testing::Value;
+
+namespace {
+
+MATCHER_P2(IntPacket, value, timestamp, "") {
+  *result_listener << "where object is (value: " << arg.template Get<int>()
+                   << ", timestamp: " << arg.Timestamp() << ")";
+  return Value(arg.template Get<int>(), Eq(value)) &&
+         Value(arg.Timestamp(), Eq(timestamp));
+}
+
+// Calculates and produces sum of all passed inputs when no more packets can be
+// expected on the input stream.
+class SummaryPacketCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"SUMMARY"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    // Makes sure there are no automatic timestamp bound updates when Process
+    // is called.
+    cc->SetTimestampOffset(TimestampDiff::Unset());
+    // Currently, only ImmediateInputStreamHandler supports "done" timestamp
+    // bound update. (ImmediateInputStreamhandler handles multiple input
+    // streams differently, so, in that case, calculator adjustments may be
+    // required.)
+    // TODO: update all input stream handlers to support "done"
+    // timestamp bound update.
+    cc->SetInputStreamHandler("ImmediateInputStreamHandler");
+    // Enables processing timestamp bound updates. For this use case we are
+    // specifically interested in "done" timestamp bound update. (E.g. when
+    // all input packet sources are closed.)
+    cc->SetProcessTimestampBounds(true);
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) final {
+    if (!kIn(cc).IsEmpty()) {
+      value_ += kIn(cc).Get();
+      value_set_ = true;
+    }
+
+    if (kOut(cc).IsClosed()) {
+      // This can happen:
+      // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g.
+      //    source calculator finished generating packets sent to kIn) and
+      //    HasNextAllowedInStream() == true (which is an often case).
+      // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still
+      //    invoke Process() with Timestamp::Max to indicate "Done" timestamp
+      //    bound update.
+      return absl::OkStatus();
+    }
+
+    // TODO: input stream holding a packet with timestamp that has
+    // no next timestamp allowed in stream should always result in
+    // InputStream::IsDone() == true.
+    if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) {
+      // `Process` may or may not be invoked for "done" timestamp bound when
+      // upstream calculator fails in `Close`. Hence, extra care is needed to
+      // identify whether the calculator needs to send output.
+      // TODO: remove when "done" timestamp bound flakiness fixed.
+      if (value_set_) {
+        // kOut(cc).Send(value_) can be used here as well, however in the case
+        // of source calculator sending inputs into kIn the resulting timestamp
+        // is not well defined (e.g. it can be the last packet timestamp or
+        // Timestamp::Max())
+        // TODO: last packet from source should always result in
+        // InputStream::IsDone() == true.
+        kOut(cc).Send(value_, Timestamp::Max());
+      }
+      kOut(cc).Close();
+    }
+    return absl::OkStatus();
+  }
+
+ private:
+  int value_ = 0;
+  bool value_set_ = false;
+};
+MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnClosingAllPacketSources) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp(11));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp(10));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  send_packet(20, Timestamp::Max());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPreStreamTimestamp) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: 'input'
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: 'IN:input'
+      output_stream: 'SUMMARY:output'
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PreStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnPostStreamTimestamp) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        input_stream: 'input'
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: 'IN:input'
+          output_stream: 'SUMMARY:output'
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class IntGeneratorCalculator : public Node {
+ public:
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return tool::StatusStop();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnSourceCalculatorCompletion) {
+  std::vector<Packet> output_packets;
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+        node {
+          calculator: "IntGeneratorCalculator"
+          output_stream: "INT:int_value"
+        }
+        node {
+          calculator: "SummaryPacketCalculator"
+          input_stream: "IN:int_value"
+          output_stream: "SUMMARY:output"
+        }
+      )pb");
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_EXPECT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+}
+
+class EmitOnCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    kOut(cc).Send(20, Timestamp(0));
+    kOut(cc).Send(10, Timestamp(1000));
+    return absl::OkStatus();
+  }
+};
+MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     ProducesSummaryPacketOnAnotherCalculatorClosure) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "EmitOnCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max())));
+
+  output_packets.clear();
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInCloseCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); }
+
+  absl::Status Close(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInCloseCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInClose) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInCloseCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  MP_ASSERT_OK(graph.CloseInputStream("input"));
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+class FailureInProcessCalculator : public Node {
+ public:
+  static constexpr Input<int> kIn{"IN"};
+  static constexpr Output<int> kOut{"INT"};
+
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  absl::Status Process(CalculatorContext* cc) final {
+    return absl::InternalError("error");
+  }
+};
+MEDIAPIPE_REGISTER_NODE(FailureInProcessCalculator);
+
+TEST(SummaryPacketCalculatorUseCaseTest,
+     DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInProcess) {
+  auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
+    input_stream: "input"
+    node {
+      calculator: "FailureInProcessCalculator"
+      input_stream: "IN:input"
+      output_stream: "INT:int_value"
+    }
+    node {
+      calculator: "SummaryPacketCalculator"
+      input_stream: "IN:int_value"
+      output_stream: "SUMMARY:output"
+    }
+  )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("output", &graph_config, &output_packets);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_THAT(output_packets, IsEmpty());
+
+  auto send_packet = [&graph](int value, Timestamp timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(value).At(timestamp)));
+  };
+
+  send_packet(10, Timestamp::PostStream());
+  EXPECT_THAT(graph.WaitUntilIdle(),
+              StatusIs(absl::StatusCode::kInternal, HasSubstr("error")));
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/framework/deps/registration.h

@@ -16,7 +16,6 @@
 #define MEDIAPIPE_DEPS_REGISTRATION_H_
 
 #include <algorithm>
-#include <cstdint>
 #include <functional>
 #include <string>
 #include <tuple>
@@ -145,6 +144,23 @@ template <typename T>
 struct WrapStatusOr<absl::StatusOr<T>> {
   using type = absl::StatusOr<T>;
 };
+
+// Defining a member of this type causes P to be ODR-used, which forces its
+// instantiation if it's a static member of a template.
+// Previously we depended on the pointer's value to determine whether the size
+// of a character array is 0 or 1, forcing it to be instantiated so the
+// compiler can determine the object's layout. But using it as a template
+// argument is more compact.
+template <auto* P>
+struct ForceStaticInstantiation {
+#ifdef _MSC_VER
+  // Just having it as the template argument does not count as a use for
+  // MSVC.
+  static constexpr bool Use() { return P != nullptr; }
+  char force_static[Use()];
+#endif  // _MSC_VER
+};
+
 }  // namespace registration_internal
 
 class NamespaceAllowlist {
@@ -162,8 +178,7 @@ class FunctionRegistry {
   FunctionRegistry(const FunctionRegistry&) = delete;
   FunctionRegistry& operator=(const FunctionRegistry&) = delete;
 
-  RegistrationToken Register(absl::string_view name, Function func,
+  RegistrationToken Register(absl::string_view name, Function func)
-                             std::string filename, uint64_t line)
       ABSL_LOCKS_EXCLUDED(lock_) {
     std::string normalized_name = GetNormalizedName(name);
     absl::WriterMutexLock lock(&lock_);
@@ -173,21 +188,10 @@ class FunctionRegistry {
     }
     if (functions_.insert(std::make_pair(normalized_name, std::move(func)))
             .second) {
-#ifndef NDEBUG
-      locations_.emplace(normalized_name,
-                         std::make_pair(std::move(filename), line));
-#endif
       return RegistrationToken(
           [this, normalized_name]() { Unregister(normalized_name); });
     }
-#ifndef NDEBUG
-    LOG(FATAL) << "Function with name " << name << " already registered."
-               << " First registration at "
-               << locations_.at(normalized_name).first << ":"
-               << locations_.at(normalized_name).second;
-#else
     LOG(FATAL) << "Function with name " << name << " already registered.";
-#endif
     return RegistrationToken([]() {});
   }
 
@@ -316,11 +320,6 @@ class FunctionRegistry {
  private:
   mutable absl::Mutex lock_;
   absl::flat_hash_map<std::string, Function> functions_ ABSL_GUARDED_BY(lock_);
-#ifndef NDEBUG
-  // Stores filename and line number for useful debug log.
-  absl::flat_hash_map<std::string, std::pair<std::string, uint32_t>> locations_
-      ABSL_GUARDED_BY(lock_);
-#endif
 
   // For names included in NamespaceAllowlist, strips the namespace.
   std::string GetAdjustedName(absl::string_view name) {
@@ -351,10 +350,8 @@ class GlobalFactoryRegistry {
 
  public:
   static RegistrationToken Register(absl::string_view name,
-                                    typename Functions::Function func,
+                                    typename Functions::Function func) {
-                                    std::string filename, uint64_t line) {
+    return functions()->Register(name, std::move(func));
-    return functions()->Register(name, std::move(func), std::move(filename),
-                                 line);
   }
 
   // Invokes the specified factory function and returns the result.
@@ -414,12 +411,77 @@ class GlobalFactoryRegistry {
 #define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \
   static auto* REGISTRY_STATIC_VAR(registration_##name, __LINE__) =  \
       new mediapipe::RegistrationToken(                              \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
 
+#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, \
+                                                      name, ...)              \
+  static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                      \
+      new mediapipe::RegistrationToken(                                       \
+          RegistryType::Register(name, __VA_ARGS__))
+
+// TODO: migrate to the above.
 #define REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, name, ...) \
   static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) =                       \
       new mediapipe::RegistrationToken(                                        \
-          RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__))
+          RegistryType::Register(#name, __VA_ARGS__))
+
+// Defines a utility registrator class which can be used to automatically
+// register factory functions.
+//
+// Example:
+// === Defining a registry ================================================
+//
+//  class Component {};
+//
+//  using ComponentRegistry = GlobalFactoryRegistry<std::unique_ptr<Component>>;
+//
+// === Defining a registrator =============================================
+//
+//  MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(ComponentRegistrator,
+//                                        ComponentRegistry, T::kName,
+//                                        absl::make_unique<T>);
+//
+// === Defining and registering a new component. ==========================
+//
+//  class MyComponent : public Component,
+//                      private ComponentRegistrator<MyComponent> {
+//   public:
+//    static constexpr char kName[] = "MyComponent";
+//    ...
+//  };
+//
+// NOTE:
+// - MyComponent is automatically registered in ComponentRegistry by
+//   "MyComponent" name.
+// - Every component is require to provide its name (T::kName here.)
+#define MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(RegistratorName, RegistryType,  \
+                                              name, ...)                      \
+  template <typename T>                                                       \
+  struct Internal##RegistratorName {                                          \
+    static NoDestructor<mediapipe::RegistrationToken> registration;           \
+                                                                              \
+    static mediapipe::RegistrationToken Make() {                              \
+      return RegistryType::Register(name, __VA_ARGS__);                       \
+    }                                                                         \
+                                                                              \
+    using RequireStatics =                                                    \
+        registration_internal::ForceStaticInstantiation<&registration>;       \
+  };                                                                          \
+  /* Static members of template classes can be defined in the header. */      \
+  template <typename T>                                                       \
+  NoDestructor<mediapipe::RegistrationToken>                                  \
+      Internal##RegistratorName<T>::registration(                             \
+          Internal##RegistratorName<T>::Make());                              \
+                                                                              \
+  template <typename T>                                                       \
+  class RegistratorName {                                                     \
+   private:                                                                   \
+    /* The member below triggers instantiation of the registration static. */ \
+    /* Note that the constructor of calculator subclasses is only invoked  */ \
+    /* through the registration token, and so we cannot simply use the     */ \
+    /* static in theconstructor.                                           */ \
+    typename Internal##RegistratorName<T>::RequireStatics register_;          \
+  };
 
 }  // namespace mediapipe
 

mediapipe/framework/deps/safe_int.h

@@ -88,10 +88,13 @@ class SafeIntStrongIntValidator {
 
     // If the argument is floating point, we can do a simple check to make
     // sure the value is in range.  It is undefined behavior to convert to int
-    // from a float that is out of range.
+    // from a float that is out of range. Since large integers will loose some
+    // precision when being converted to floating point, the integer max and min
+    // are explicitly converted back to floating point for this comparison, in
+    // order to satisfy compiler warnings.
     if (std::is_floating_point<U>::value) {
-      if (arg < std::numeric_limits<T>::min() ||
+      if (arg < static_cast<U>(std::numeric_limits<T>::min()) ||
-          arg > std::numeric_limits<T>::max()) {
+          arg > static_cast<U>(std::numeric_limits<T>::max())) {
         ErrorType::Error("SafeInt: init from out of bounds float", arg, "=");
       }
     } else {
@@ -284,11 +287,11 @@ class SafeIntStrongIntValidator {
 // A SafeIntStrongIntValidator policy class to LOG(FATAL) on errors.
 struct LogFatalOnError {
   template <typename Tlhs, typename Trhs>
-  static void Error(const char *error, Tlhs lhs, Trhs rhs, const char *op) {
+  static void Error(const char* error, Tlhs lhs, Trhs rhs, const char* op) {
     LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")";
   }
   template <typename Tval>
-  static void Error(const char *error, Tval val, const char *op) {
+  static void Error(const char* error, Tval val, const char* op) {
     LOG(FATAL) << error << ": (" << op << val << ")";
   }
 };

mediapipe/framework/deps/status_builder.cc

@@ -68,11 +68,11 @@ StatusBuilder&& StatusBuilder::SetNoLogging() && {
   return std::move(SetNoLogging());
 }
 
-StatusBuilder::operator Status() const& {
+StatusBuilder::operator absl::Status() const& {
   return StatusBuilder(*this).JoinMessageToStatus();
 }
 
-StatusBuilder::operator Status() && { return JoinMessageToStatus(); }
+StatusBuilder::operator absl::Status() && { return JoinMessageToStatus(); }
 
 absl::Status StatusBuilder::JoinMessageToStatus() {
   if (!impl_) {

mediapipe/framework/deps/status_builder.h

@@ -83,8 +83,8 @@ class ABSL_MUST_USE_RESULT StatusBuilder {
     return std::move(*this << msg);
   }
 
-  operator Status() const&;
+  operator absl::Status() const&;
-  operator Status() &&;
+  operator absl::Status() &&;
 
   absl::Status JoinMessageToStatus();
 

mediapipe/framework/deps/strong_int.h

@@ -403,11 +403,11 @@ std::ostream &operator<<(std::ostream &os,
     lhs op## = rhs;                                                       \
     return lhs;                                                           \
   }
-STRONG_INT_VS_STRONG_INT_BINARY_OP(+);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(+)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(-);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(-)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(&);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(&)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(|);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(|)
-STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
+STRONG_INT_VS_STRONG_INT_BINARY_OP(^)
 #undef STRONG_INT_VS_STRONG_INT_BINARY_OP
 
 // Define operators that take one StrongInt and one native integer argument.
@@ -431,12 +431,12 @@ STRONG_INT_VS_STRONG_INT_BINARY_OP(^);
     rhs op## = lhs;                                                       \
     return rhs;                                                           \
   }
-STRONG_INT_VS_NUMERIC_BINARY_OP(*);
+STRONG_INT_VS_NUMERIC_BINARY_OP(*)
-NUMERIC_VS_STRONG_INT_BINARY_OP(*);
+NUMERIC_VS_STRONG_INT_BINARY_OP(*)
-STRONG_INT_VS_NUMERIC_BINARY_OP(/);
+STRONG_INT_VS_NUMERIC_BINARY_OP(/)
-STRONG_INT_VS_NUMERIC_BINARY_OP(%);
+STRONG_INT_VS_NUMERIC_BINARY_OP(%)
-STRONG_INT_VS_NUMERIC_BINARY_OP(<<);  // NOLINT(whitespace/operators)
+STRONG_INT_VS_NUMERIC_BINARY_OP(<<)  // NOLINT(whitespace/operators)
-STRONG_INT_VS_NUMERIC_BINARY_OP(>>);  // NOLINT(whitespace/operators)
+STRONG_INT_VS_NUMERIC_BINARY_OP(>>)  // NOLINT(whitespace/operators)
 #undef STRONG_INT_VS_NUMERIC_BINARY_OP
 #undef NUMERIC_VS_STRONG_INT_BINARY_OP
 
@@ -447,12 +447,12 @@ STRONG_INT_VS_NUMERIC_BINARY_OP(>>);  // NOLINT(whitespace/operators)
                           StrongInt<TagType, ValueType, ValidatorType> rhs) { \
     return lhs.value() op rhs.value();                                        \
   }
-STRONG_INT_COMPARISON_OP(==);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(==)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(!=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(!=)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(<);   // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(<)   // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(<=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(<=)  // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(>);   // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(>)   // NOLINT(whitespace/operators)
-STRONG_INT_COMPARISON_OP(>=);  // NOLINT(whitespace/operators)
+STRONG_INT_COMPARISON_OP(>=)  // NOLINT(whitespace/operators)
 #undef STRONG_INT_COMPARISON_OP
 
 }  // namespace intops

mediapipe/framework/encode_binary_proto.bzl

@@ -37,29 +37,33 @@ Args:
   output: The desired name of the output file. Optional.
 """
 
+load("@bazel_skylib//lib:paths.bzl", "paths")
+
 PROTOC = "@com_google_protobuf//:protoc"
 
-def _canonicalize_proto_path_oss(all_protos, genfile_path):
+def _canonicalize_proto_path_oss(f):
-    """For the protos from external repository, canonicalize the proto path and the file name.
+    if not f.root.path:
+        return struct(
+            proto_path = ".",
+            file_name = f.short_path,
+        )
 
-    Returns:
+    # `f.path` looks like "<genfiles>/external/<repo>/(_virtual_imports/<library>/)?<file_name>"
-       Proto path list and proto source file list.
+    repo_name, _, file_name = f.path[len(paths.join(f.root.path, "external") + "/"):].partition("/")
-    """
+    if file_name.startswith("_virtual_imports/"):
-    proto_paths = []
+        # This is a virtual import; move "_virtual_imports/<library>" from `repo_name` to `file_name`.
-    proto_file_names = []
+        repo_name = paths.join(repo_name, *file_name.split("/", 2)[:2])
-    for s in all_protos.to_list():
+        file_name = file_name.split("/", 2)[-1]
-        if s.path.startswith(genfile_path):
+    return struct(
-            repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/")
+        proto_path = paths.join(f.root.path, "external", repo_name),
+        file_name = file_name,
+    )
 
-            # handle virtual imports
+def _map_root_path(f):
-            if file_name.startswith("_virtual_imports"):
+    return _canonicalize_proto_path_oss(f).proto_path
-                repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2])
+
-                file_name = file_name.split("/", 2)[-1]
+def _map_short_path(f):
-            proto_paths.append(genfile_path + "/external/" + repo_name)
+    return _canonicalize_proto_path_oss(f).file_name
-            proto_file_names.append(file_name)
-        else:
-            proto_file_names.append(s.path)
-    return ([" --proto_path=" + path for path in proto_paths], proto_file_names)
 
 def _get_proto_provider(dep):
     """Get the provider for protocol buffers from a dependnecy.
@@ -90,24 +94,35 @@ def _encode_binary_proto_impl(ctx):
         sibling = textpb,
     )
 
-    path_list, file_list = _canonicalize_proto_path_oss(all_protos, ctx.genfiles_dir.path)
+    args = ctx.actions.args()
+    args.add(textpb)
+    args.add(binarypb)
+    args.add(ctx.executable._proto_compiler)
+    args.add(ctx.attr.message_type, format = "--encode=%s")
+    args.add("--proto_path=.")
+    args.add_all(
+        all_protos,
+        map_each = _map_root_path,
+        format_each = "--proto_path=%s",
+        uniquify = True,
+    )
+    args.add_all(
+        all_protos,
+        map_each = _map_short_path,
+        uniquify = True,
+    )
 
     # Note: the combination of absolute_paths and proto_path, as well as the exact
     # order of gendir before ., is needed for the proto compiler to resolve
     # import statements that reference proto files produced by a genrule.
     ctx.actions.run_shell(
-        tools = all_protos.to_list() + [textpb, ctx.executable._proto_compiler],
+        tools = depset(
-        outputs = [binarypb],
+            direct = [textpb, ctx.executable._proto_compiler],
-        command = " ".join(
+            transitive = [all_protos],
-            [
-                ctx.executable._proto_compiler.path,
-                "--encode=" + ctx.attr.message_type,
-                "--proto_path=" + ctx.genfiles_dir.path,
-                "--proto_path=" + ctx.bin_dir.path,
-                "--proto_path=.",
-            ] + path_list + file_list +
-            ["<", textpb.path, ">", binarypb.path],
         ),
+        outputs = [binarypb],
+        command = "${@:3} < $1 > $2",
+        arguments = [args],
         mnemonic = "EncodeProto",
     )
 

mediapipe/framework/formats/BUILD

@@ -481,6 +481,7 @@ cc_library(
 cc_test(
     name = "tensor_test",
     srcs = ["tensor_test.cc"],
+    tags = ["not_run:arm"],
     deps = [
         ":tensor",
         "//mediapipe/framework/port:gtest_main",

mediapipe/framework/formats/body_rig.proto

@@ -19,7 +19,7 @@ package mediapipe;
 // Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of
 // the joint and its visibility.
 message Joint {
-  // Joint rotation in 6D contineous representation ordered as
+  // Joint rotation in 6D continuous representation ordered as
   // [a1, b1, a2, b2, a3, b3].
   //
   // Such representation is more sutable for NN model training and can be

mediapipe/framework/formats/tensor.cc

@@ -117,8 +117,8 @@ MtlBufferView MtlBufferView::GetReadView(const Tensor& tensor,
       << "Tensor must be written prior to read from.";
   LOG_IF(FATAL,
          !(tensor.valid_ & (Tensor::kValidCpu | Tensor::kValidMetalBuffer)))
-      << "Tensor conversion between different GPU resources is not supported "
+      << "Tensor conversion between different GPU backing formats is not "
-         "yet.";
+         "supported yet.";
   auto lock(absl::make_unique<absl::MutexLock>(&tensor.view_mutex_));
   tensor.valid_ |= Tensor::kValidMetalBuffer;
   AllocateMtlBuffer(tensor, [command_buffer device]);
@@ -164,8 +164,8 @@ Tensor::OpenGlTexture2dView Tensor::GetOpenGlTexture2dReadView() const {
   LOG_IF(FATAL, valid_ == kValidNone)
       << "Tensor must be written prior to read from.";
   LOG_IF(FATAL, !(valid_ & (kValidCpu | kValidOpenGlTexture2d)))
-      << "Tensor conversion between different GPU resources is not supported "
+      << "Tensor conversion between different GPU backing formats is not "
-         "yet.";
+         "supported yet.";
   auto lock = absl::make_unique<absl::MutexLock>(&view_mutex_);
   AllocateOpenGlTexture2d();
   if (!(valid_ & kValidOpenGlTexture2d)) {
@@ -335,7 +335,8 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
                             kValidAHardwareBuffer |
 #endif  // MEDIAPIPE_TENSOR_USE_AHWB
                             kValidOpenGlBuffer)))
-      << "Tensor conversion between different GPU resources is not supported.";
+      << "Tensor conversion between different GPU backing formats is not "
+         "supported yet.";
   auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
   AllocateOpenGlBuffer();
   if (!(valid_ & kValidOpenGlBuffer)) {

mediapipe/framework/formats/tensor.h

@@ -117,11 +117,18 @@ class Tensor {
     Shape() = default;
     Shape(std::initializer_list<int> dimensions) : dims(dimensions) {}
     Shape(const std::vector<int>& dimensions) : dims(dimensions) {}
+    Shape(std::initializer_list<int> dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
+    Shape(const std::vector<int>& dimensions, bool is_dynamic)
+        : dims(dimensions), is_dynamic(is_dynamic) {}
     int num_elements() const {
       return std::accumulate(dims.begin(), dims.end(), 1,
                              std::multiplies<int>());
     }
     std::vector<int> dims;
+    // The Tensor has dynamic rather than static shape so the TFLite interpreter
+    // needs to be reallocated. Only relevant for CPU.
+    bool is_dynamic = false;
   };
   // Quantization parameters corresponding to the zero_point and scale value
   // made available by TfLite quantized (uint8/int8) tensors.

mediapipe/framework/formats/tensor_test.cc

@@ -2,6 +2,7 @@
 
 #include <cstring>
 #include <string>
+#include <vector>
 
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
@@ -34,6 +35,17 @@ TEST(General, TestDataTypes) {
   EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool));
 }
 
+TEST(General, TestDynamic) {
+  Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true));
+  EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4);
+  EXPECT_TRUE(t1.shape().is_dynamic);
+
+  std::vector<int> t2_dims = {4, 3, 2, 3};
+  Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true));
+  EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3);
+  EXPECT_TRUE(t2.shape().is_dynamic);
+}
+
 TEST(Cpu, TestMemoryAllocation) {
   Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3});
   auto v1 = t1.GetCpuWriteView();

mediapipe/framework/graph_service.h

@@ -44,7 +44,6 @@ class GraphServiceBase {
 
   constexpr GraphServiceBase(const char* key) : key(key) {}
 
-  virtual ~GraphServiceBase() = default;
   inline virtual absl::StatusOr<Packet> CreateDefaultObject() const {
     return DefaultInitializationUnsupported();
   }
@@ -52,14 +51,32 @@ class GraphServiceBase {
   const char* key;
 
  protected:
+  // `GraphService<T>` objects, deriving `GraphServiceBase` are designed to be
+  // global constants and not ever deleted through `GraphServiceBase`. Hence,
+  // protected and non-virtual destructor which helps to make `GraphService<T>`
+  // trivially destructible and properly defined as global constants.
+  //
+  // A class with any virtual functions should have a destructor that is either
+  // public and virtual or else protected and non-virtual.
+  // https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rc-dtor-virtual
+  ~GraphServiceBase() = default;
+
   absl::Status DefaultInitializationUnsupported() const {
     return absl::UnimplementedError(absl::StrCat(
         "Graph service '", key, "' does not support default initialization"));
   }
 };
 
+// A global constant to refer a service:
+// - Requesting `CalculatorContract::UseService` from calculator
+// - Accessing `Calculator/SubgraphContext::Service`from calculator/subgraph
+// - Setting before graph initialization `CalculatorGraph::SetServiceObject`
+//
+// NOTE: In headers, define your graph service reference safely as following:
+// `inline constexpr GraphService<YourService> kYourService("YourService");`
+//
 template <typename T>
-class GraphService : public GraphServiceBase {
+class GraphService final : public GraphServiceBase {
  public:
   using type = T;
   using packet_type = std::shared_ptr<T>;
@@ -68,7 +85,7 @@ class GraphService : public GraphServiceBase {
                                                  kDisallowDefaultInitialization)
       : GraphServiceBase(my_key), default_init_(default_init) {}
 
-  absl::StatusOr<Packet> CreateDefaultObject() const override {
+  absl::StatusOr<Packet> CreateDefaultObject() const final {
     if (default_init_ != kAllowDefaultInitialization) {
       return DefaultInitializationUnsupported();
     }