Project import generated by Copybara.

GitOrigin-RevId: 2146b10f0a498f665f246e16033b686c7947b92d

mediapipe/calculators/core/BUILD

@@ -233,6 +233,22 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "concatenate_vector_calculator_hdr",
+    hdrs = ["concatenate_vector_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":concatenate_vector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "concatenate_vector_calculator",
     srcs = ["concatenate_vector_calculator.cc"],

mediapipe/calculators/core/default_side_packet_calculator.cc

@@ -71,7 +71,8 @@ absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) {
   if (cc->InputSidePackets().HasTag(kOptionalValueTag)) {
     cc->InputSidePackets()
         .Tag(kOptionalValueTag)
-        .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag));
+        .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag))
+        .Optional();
   }
 
   RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag));

mediapipe/calculators/image/BUILD

@@ -410,7 +410,9 @@ cc_library(
     srcs = ["image_properties_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework/api2:node",
         "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",

mediapipe/calculators/image/image_properties_calculator.cc

@@ -12,25 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/image_frame.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gpu_buffer.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-namespace {
-constexpr char kImageFrameTag[] = "IMAGE";
-constexpr char kGpuBufferTag[] = "IMAGE_GPU";
-}  // namespace
-
 namespace mediapipe {
+namespace api2 {
+
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+using GpuBuffer = AnyType;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
 
 // Extracts image properties from the input image and outputs the properties.
 // Currently only supports image size.
 // Input:
 //   One of the following:
-//   IMAGE: An ImageFrame
+//   IMAGE: An Image or ImageFrame (for backward compatibility with existing
+//          graphs that use IMAGE for ImageFrame input)
+//   IMAGE_CPU: An ImageFrame
 //   IMAGE_GPU: A GpuBuffer
 //
 // Output:
@@ -42,59 +49,64 @@ namespace mediapipe {
 //   input_stream: "IMAGE:image"
 //   output_stream: "SIZE:size"
 // }
-class ImagePropertiesCalculator : public CalculatorBase {
+class ImagePropertiesCalculator : public Node {
  public:
-  static absl::Status GetContract(CalculatorContract* cc) {
-    RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
-              cc->Inputs().HasTag(kGpuBufferTag));
-    if (cc->Inputs().HasTag(kImageFrameTag)) {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
-    }
-#if !MEDIAPIPE_DISABLE_GPU
-    if (cc->Inputs().HasTag(kGpuBufferTag)) {
-      cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
-    }
-#endif  // !MEDIAPIPE_DISABLE_GPU
+  static constexpr Input<
+      OneOf<mediapipe::Image, mediapipe::ImageFrame>>::Optional kIn{"IMAGE"};
+  // IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level
+  // graphs for the Python Solution APIs to figure out the type of input stream
+  // without running into ambiguities from IMAGE.
+  // TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image.
+  static constexpr Input<mediapipe::ImageFrame>::Optional kInCpu{"IMAGE_CPU"};
+  static constexpr Input<GpuBuffer>::Optional kInGpu{"IMAGE_GPU"};
+  static constexpr Output<std::pair<int, int>> kOut{"SIZE"};
 
-    if (cc->Outputs().HasTag("SIZE")) {
-      cc->Outputs().Tag("SIZE").Set<std::pair<int, int>>();
-    }
+  MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut);
 
-    return absl::OkStatus();
-  }
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() +
+                     kInGpu(cc).IsConnected(),
+                 1)
+        << "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is "
+           "expected.";
 
-  absl::Status Open(CalculatorContext* cc) override {
-    cc->SetOffset(TimestampDiff(0));
     return absl::OkStatus();
   }
 
   absl::Status Process(CalculatorContext* cc) override {
-    int width;
-    int height;
+    std::pair<int, int> size;
 
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
-        !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
-      const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-      width = image.Width();
-      height = image.Height();
+    if (kIn(cc).IsConnected()) {
+      kIn(cc).Visit(
+          [&size](const mediapipe::Image& value) {
+            size.first = value.width();
+            size.second = value.height();
+          },
+          [&size](const mediapipe::ImageFrame& value) {
+            size.first = value.Width();
+            size.second = value.Height();
+          });
+    }
+    if (kInCpu(cc).IsConnected()) {
+      const auto& image = *kInCpu(cc);
+      size.first = image.Width();
+      size.second = image.Height();
     }
 #if !MEDIAPIPE_DISABLE_GPU
-    if (cc->Inputs().HasTag(kGpuBufferTag) &&
-        !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
-      const auto& image =
-          cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
-      width = image.width();
-      height = image.height();
+    if (kInGpu(cc).IsConnected()) {
+      const auto& image = *kInGpu(cc);
+      size.first = image.width();
+      size.second = image.height();
     }
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
-    cc->Outputs().Tag("SIZE").AddPacket(
-        MakePacket<std::pair<int, int>>(width, height)
-            .At(cc->InputTimestamp()));
+    kOut(cc).Send(size);
 
     return absl::OkStatus();
   }
 };
-REGISTER_CALCULATOR(ImagePropertiesCalculator);
 
+MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/tensor/BUILD

@@ -585,6 +585,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
+    visibility = ["//visibility:public"],
     deps = [
         ":image_to_tensor_utils",
         "//mediapipe/framework/formats:image",

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter {
       return absl::OkStatus();
     }));
 
-    return tensor;
+    return std::move(tensor);
   }
 
   ~GlProcessor() override {

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
           tflite::gpu::HW(output_dims.height, output_dims.width),
           command_buffer, buffer_view.buffer()));
       [command_buffer commit];
-      return tensor;
+      return std::move(tensor);
     }
   }
 

mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc

@@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
         GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
                                     range_min, range_max));
     transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset);
-    return tensor;
+    return std::move(tensor);
   }
 
  private:

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -205,11 +205,12 @@ class VelocityFilter : public LandmarksFilter {
 class OneEuroFilterImpl : public LandmarksFilter {
  public:
   OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
-                    double derivate_cutoff)
+                    double derivate_cutoff, float min_allowed_object_scale)
       : frequency_(frequency),
         min_cutoff_(min_cutoff),
         beta_(beta),
-        derivate_cutoff_(derivate_cutoff) {}
+        derivate_cutoff_(derivate_cutoff),
+        min_allowed_object_scale_(min_allowed_object_scale) {}
 
   absl::Status Reset() override {
     x_filters_.clear();
@@ -224,15 +225,25 @@ class OneEuroFilterImpl : public LandmarksFilter {
     // Initialize filters once.
     MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
 
+    const float object_scale = GetObjectScale(in_landmarks);
+    if (object_scale < min_allowed_object_scale_) {
+      *out_landmarks = in_landmarks;
+      return absl::OkStatus();
+    }
+    const float value_scale = 1.0f / object_scale;
+
     // Filter landmarks. Every axis of every landmark is filtered separately.
     for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
       const auto& in_landmark = in_landmarks.landmark(i);
 
       auto* out_landmark = out_landmarks->add_landmark();
       *out_landmark = in_landmark;
-      out_landmark->set_x(x_filters_[i].Apply(timestamp, in_landmark.x()));
-      out_landmark->set_y(y_filters_[i].Apply(timestamp, in_landmark.y()));
-      out_landmark->set_z(z_filters_[i].Apply(timestamp, in_landmark.z()));
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
     }
 
     return absl::OkStatus();
@@ -265,6 +276,7 @@ class OneEuroFilterImpl : public LandmarksFilter {
   double min_cutoff_;
   double beta_;
   double derivate_cutoff_;
+  double min_allowed_object_scale_;
 
   std::vector<OneEuroFilter> x_filters_;
   std::vector<OneEuroFilter> y_filters_;
@@ -344,7 +356,8 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
         options.one_euro_filter().frequency(),
         options.one_euro_filter().min_cutoff(),
         options.one_euro_filter().beta(),
-        options.one_euro_filter().derivate_cutoff());
+        options.one_euro_filter().derivate_cutoff(),
+        options.one_euro_filter().min_allowed_object_scale());
   } else {
     RET_CHECK_FAIL()
         << "Landmarks filter is either not specified or not supported";

mediapipe/calculators/util/landmarks_smoothing_calculator.proto

@@ -50,9 +50,9 @@ message LandmarksSmoothingCalculatorOptions {
   // For the details of the filter implementation and the procedure of its
   // configuration please check http://cristal.univ-lille.fr/~casiez/1euro/
   message OneEuroFilter {
-    // Frequency of incomming frames defined in seconds. Used only if can't be
-    // calculated from provided events (e.g. on the very first frame).
-    optional float frequency = 1 [default = 0.033];
+    // Frequency of incomming frames defined in frames per seconds. Used only if
+    // can't be calculated from provided events (e.g. on the very first frame).
+    optional float frequency = 1 [default = 30.0];
 
     // Minimum cutoff frequency. Start by tuning this parameter while keeping
     // `beta = 0` to reduce jittering to the desired level. 1Hz (the default
@@ -68,6 +68,10 @@ message LandmarksSmoothingCalculatorOptions {
     // algorithm, but can be tuned to further smooth the speed (i.e. derivate)
     // on the object.
     optional float derivate_cutoff = 4 [default = 1.0];
+
+    // If calculated object scale is less than given value smoothing will be
+    // disabled and landmarks will be returned as is.
+    optional float min_allowed_object_scale = 5 [default = 1e-6];
   }
 
   oneof filter_options {

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc

@@ -77,10 +77,12 @@ class RefineLandmarksFromHeatmapCalculatorImpl
     const auto& options =
         cc->Options<mediapipe::RefineLandmarksFromHeatmapCalculatorOptions>();
 
-    ASSIGN_OR_RETURN(auto out_lms, RefineLandmarksFromHeatMap(
-                                       in_lms, hm_raw, hm_tensor.shape().dims,
-                                       options.kernel_size(),
-                                       options.min_confidence_to_refine()));
+    ASSIGN_OR_RETURN(
+        auto out_lms,
+        RefineLandmarksFromHeatMap(
+            in_lms, hm_raw, hm_tensor.shape().dims, options.kernel_size(),
+            options.min_confidence_to_refine(), options.refine_presence(),
+            options.refine_visibility()));
 
     kOutLandmarks(cc).Send(std::move(out_lms));
     return absl::OkStatus();
@@ -104,7 +106,8 @@ class RefineLandmarksFromHeatmapCalculatorImpl
 absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     const mediapipe::NormalizedLandmarkList& in_lms,
     const float* heatmap_raw_data, const std::vector<int>& heatmap_dims,
-    int kernel_size, float min_confidence_to_refine) {
+    int kernel_size, float min_confidence_to_refine, bool refine_presence,
+    bool refine_visibility) {
   ASSIGN_OR_RETURN(auto hm_dims, GetHwcFromDims(heatmap_dims));
   auto [hm_height, hm_width, hm_channels] = hm_dims;
 
@@ -136,7 +139,7 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     float sum = 0;
     float weighted_col = 0;
     float weighted_row = 0;
-    float max_value = 0;
+    float max_confidence_value = 0;
 
     // Main loop. Go over kernel and calculate weighted sum of coordinates,
     // sum of weights and max weights.
@@ -150,15 +153,33 @@ absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
         // options.
         float confidence = Sigmoid(heatmap_raw_data[idx]);
         sum += confidence;
-        max_value = std::max(max_value, confidence);
+        max_confidence_value = std::max(max_confidence_value, confidence);
         weighted_col += col * confidence;
         weighted_row += row * confidence;
       }
     }
-    if (max_value >= min_confidence_to_refine && sum > 0) {
+    if (max_confidence_value >= min_confidence_to_refine && sum > 0) {
       out_lms.mutable_landmark(lm_index)->set_x(weighted_col / hm_width / sum);
       out_lms.mutable_landmark(lm_index)->set_y(weighted_row / hm_height / sum);
     }
+    if (refine_presence && sum > 0 &&
+        out_lms.landmark(lm_index).has_presence()) {
+      // We assume confidence in heatmaps describes landmark presence.
+      // If landmark is not confident in heatmaps, probably it is not present.
+      const float presence = out_lms.landmark(lm_index).presence();
+      const float new_presence = std::min(presence, max_confidence_value);
+      out_lms.mutable_landmark(lm_index)->set_presence(new_presence);
+    }
+    if (refine_visibility && sum > 0 &&
+        out_lms.landmark(lm_index).has_visibility()) {
+      // We assume confidence in heatmaps describes landmark presence.
+      // As visibility = (not occluded but still present) -> that mean that if
+      // landmark is not present, it is not visible as well.
+      // I.e. visibility confidence cannot be bigger than presence confidence.
+      const float visibility = out_lms.landmark(lm_index).visibility();
+      const float new_visibility = std::min(visibility, max_confidence_value);
+      out_lms.mutable_landmark(lm_index)->set_visibility(new_visibility);
+    }
   }
   return out_lms;
 }

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h

@@ -43,7 +43,8 @@ class RefineLandmarksFromHeatmapCalculator : public NodeIntf {
 absl::StatusOr<mediapipe::NormalizedLandmarkList> RefineLandmarksFromHeatMap(
     const mediapipe::NormalizedLandmarkList& in_lms,
     const float* heatmap_raw_data, const std::vector<int>& heatmap_dims,
-    int kernel_size, float min_confidence_to_refine);
+    int kernel_size, float min_confidence_to_refine, bool refine_presence,
+    bool refine_visibility);
 
 }  // namespace mediapipe
 

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto

@@ -24,4 +24,6 @@ message RefineLandmarksFromHeatmapCalculatorOptions {
   }
   optional int32 kernel_size = 1 [default = 9];
   optional float min_confidence_to_refine = 2 [default = 0.5];
+  optional bool refine_presence = 3 [default = false];
+  optional bool refine_visibility = 4 [default = false];
 }

mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc

@@ -70,8 +70,8 @@ TEST(RefineLandmarksFromHeatmapTest, Smoke) {
     z, z, z};
   // clang-format on
 
-  auto ret_or_error = RefineLandmarksFromHeatMap(vec_to_lms({{0.5, 0.5}}),
-                                                 hm.data(), {3, 3, 1}, 3, 0.1);
+  auto ret_or_error = RefineLandmarksFromHeatMap(
+      vec_to_lms({{0.5, 0.5}}), hm.data(), {3, 3, 1}, 3, 0.1, true, true);
   MP_EXPECT_OK(ret_or_error);
   EXPECT_THAT(lms_to_vec(*ret_or_error),
               ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.))));
@@ -94,7 +94,7 @@ TEST(RefineLandmarksFromHeatmapTest, MultiLayer) {
 
   auto ret_or_error = RefineLandmarksFromHeatMap(
       vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3,
-      0.1);
+      0.1, true, true);
   MP_EXPECT_OK(ret_or_error);
   EXPECT_THAT(lms_to_vec(*ret_or_error),
               ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)),
@@ -119,7 +119,7 @@ TEST(RefineLandmarksFromHeatmapTest, KeepIfNotSure) {
 
   auto ret_or_error = RefineLandmarksFromHeatMap(
       vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3,
-      0.6);
+      0.6, true, true);
   MP_EXPECT_OK(ret_or_error);
   EXPECT_THAT(lms_to_vec(*ret_or_error),
               ElementsAre(Pair(FloatEq(0.5), FloatEq(0.5)),
@@ -140,8 +140,9 @@ TEST(RefineLandmarksFromHeatmapTest, Border) {
     z, z, 0}, 3, 3, 2);
   // clang-format on
 
-  auto ret_or_error = RefineLandmarksFromHeatMap(
-      vec_to_lms({{0.0, 0.0}, {0.9, 0.9}}), hm.data(), {3, 3, 2}, 3, 0.1);
+  auto ret_or_error =
+      RefineLandmarksFromHeatMap(vec_to_lms({{0.0, 0.0}, {0.9, 0.9}}),
+                                 hm.data(), {3, 3, 2}, 3, 0.1, true, true);
   MP_EXPECT_OK(ret_or_error);
   EXPECT_THAT(lms_to_vec(*ret_or_error),
               ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)),