Project import generated by Copybara.

GitOrigin-RevId: 373e3ac1e5839befd95bf7d73ceff3c5f1171969

mediapipe/calculators/core/flow_limiter_calculator.proto

@@ -30,7 +30,7 @@ message FlowLimiterCalculatorOptions {
   optional int32 max_in_flight = 1 [default = 1];
 
   // The maximum number of frames queued waiting for processing.
-  // The default value limits to 1 frame awaiting processing.
+  // The default value limits to 0 frames awaiting processing.
   optional int32 max_in_queue = 2 [default = 0];
 
   // The maximum time in microseconds to wait for a frame to finish processing.

mediapipe/calculators/core/split_vector_calculator.cc

@@ -80,4 +80,7 @@ typedef SplitVectorCalculator<mediapipe::ClassificationList, false>
     SplitClassificationListVectorCalculator;
 REGISTER_CALCULATOR(SplitClassificationListVectorCalculator);
 
+typedef SplitVectorCalculator<uint64_t, false> SplitUint64tVectorCalculator;
+REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -480,8 +480,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
   if (cc->Inputs().HasTag(kRectTag)) {
     const auto& rect = cc->Inputs().Tag(kRectTag).Get<Rect>();
     // Only use the rect if it is valid.
-    if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 &&
-        rect.y_center() >= 0) {
+    if (rect.width() > 0 && rect.height() > 0) {
       x_center = rect.x_center();
       y_center = rect.y_center();
       crop_width = rect.width();

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -337,12 +337,15 @@ absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) {
       !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
     flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
   }
-  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS") &&
-      !cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
-    const auto& image_size =
-        cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
-    output_width_ = image_size.first;
-    output_height_ = image_size.second;
+  if (cc->Inputs().HasTag("OUTPUT_DIMENSIONS")) {
+    if (cc->Inputs().Tag("OUTPUT_DIMENSIONS").IsEmpty()) {
+      return absl::OkStatus();
+    } else {
+      const auto& image_size =
+          cc->Inputs().Tag("OUTPUT_DIMENSIONS").Get<std::pair<int, int>>();
+      output_width_ = image_size.first;
+      output_height_ = image_size.second;
+    }
   }
 
   if (use_gpu_) {
@@ -506,6 +509,14 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
   ComputeOutputDimensions(input_width, input_height, &output_width,
                           &output_height);
 
+  if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
+    const float scale =
+        std::min(static_cast<float>(output_width_) / input_width,
+                 static_cast<float>(output_height_) / input_height);
+    output_width = std::round(input_width * scale);
+    output_height = std::round(input_height * scale);
+  }
+
   if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
     auto padding = absl::make_unique<std::array<float, 4>>();
     ComputeOutputLetterboxPadding(input_width, input_height, output_width,

mediapipe/calculators/image/set_alpha_calculator.cc

@@ -53,7 +53,7 @@ enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 // The alpha channel can be set to a single value, or come from an image mask.
 // If the input image has an alpha channel, it will be updated.
 // If the input image doesn't have an alpha channel, one will be added.
-// Adding alpha channel to a Grayscale (single channel) input is not suported.
+// Adding alpha channel to a Grayscale (single channel) input is not supported.
 //
 // Inputs:
 //   One of the following two IMAGE tags:

mediapipe/calculators/util/BUILD

@@ -1384,6 +1384,32 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_refinement_calculator_proto",
+    srcs = ["landmarks_refinement_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_refinement_calculator",
+    srcs = ["landmarks_refinement_calculator.cc"],
+    hdrs = ["landmarks_refinement_calculator.h"],
+    deps = [
+        ":landmarks_refinement_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:core_proto",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
 cc_test(
     name = "refine_landmarks_from_heatmap_calculator_test",
     srcs = ["refine_landmarks_from_heatmap_calculator_test.cc"],

mediapipe/calculators/util/landmarks_refinement_calculator.cc

@@ -0,0 +1,197 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/landmarks_refinement_calculator.h"
+
+#include <algorithm>
+#include <set>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/util/landmarks_refinement_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/proto_ns.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+namespace api2 {
+
+namespace {
+
+absl::StatusOr<int> GetNumberOfRefinedLandmarks(
+    const proto_ns::RepeatedPtrField<
+        LandmarksRefinementCalculatorOptions::Refinement>& refinements) {
+  // Gather all used indexes.
+  std::set<int> idxs;
+  for (int i = 0; i < refinements.size(); ++i) {
+    const auto& refinement = refinements.Get(i);
+    for (int i = 0; i < refinement.indexes_mapping_size(); ++i) {
+      idxs.insert(refinement.indexes_mapping(i));
+    }
+  }
+
+  // Check that indxes start with 0 and there is no gaps between min and max
+  // indexes.
+  RET_CHECK(!idxs.empty())
+      << "There should be at least one landmark in indexes mapping";
+  int idxs_min = *idxs.begin();
+  int idxs_max = *idxs.rbegin();
+  int n_idxs = idxs.size();
+  RET_CHECK_EQ(idxs_min, 0)
+      << "Indexes are expected to start with 0 instead of " << idxs_min;
+  RET_CHECK_EQ(idxs_max, n_idxs - 1)
+      << "Indexes should have no gaps but " << idxs_max - n_idxs + 1
+      << " indexes are missing";
+
+  return n_idxs;
+}
+
+void RefineXY(const proto_ns::RepeatedField<int>& indexes_mapping,
+              const NormalizedLandmarkList& landmarks,
+              NormalizedLandmarkList* refined_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+    auto* refined_landmark =
+        refined_landmarks->mutable_landmark(indexes_mapping.Get(i));
+    refined_landmark->set_x(landmark.x());
+    refined_landmark->set_y(landmark.y());
+  }
+}
+
+float GetZAverage(const NormalizedLandmarkList& landmarks,
+                  const proto_ns::RepeatedField<int>& indexes) {
+  double z_sum = 0;
+  for (int i = 0; i < indexes.size(); ++i) {
+    z_sum += landmarks.landmark(indexes.Get(i)).z();
+  }
+  return z_sum / indexes.size();
+}
+
+void RefineZ(
+    const proto_ns::RepeatedField<int>& indexes_mapping,
+    const LandmarksRefinementCalculatorOptions::ZRefinement& z_refinement,
+    const NormalizedLandmarkList& landmarks,
+    NormalizedLandmarkList* refined_landmarks) {
+  if (z_refinement.has_none()) {
+    // Do nothing and keep Z that is already in refined landmarks.
+  } else if (z_refinement.has_copy()) {
+    for (int i = 0; i < landmarks.landmark_size(); ++i) {
+      refined_landmarks->mutable_landmark(indexes_mapping.Get(i))
+          ->set_z(landmarks.landmark(i).z());
+    }
+  } else if (z_refinement.has_assign_average()) {
+    const float z_average =
+        GetZAverage(*refined_landmarks,
+                    z_refinement.assign_average().indexes_for_average());
+    for (int i = 0; i < indexes_mapping.size(); ++i) {
+      refined_landmarks->mutable_landmark(indexes_mapping.Get(i))
+          ->set_z(z_average);
+    }
+  } else {
+    CHECK(false) << "Z refinement is either not specified or not supported";
+  }
+}
+
+}  // namespace
+
+class LandmarksRefinementCalculatorImpl
+    : public NodeImpl<LandmarksRefinementCalculator> {
+  absl::Status Open(CalculatorContext* cc) override {
+    options_ = cc->Options<LandmarksRefinementCalculatorOptions>();
+
+    // Validate refinements.
+    for (int i = 0; i < options_.refinement_size(); ++i) {
+      const auto& refinement = options_.refinement(i);
+      RET_CHECK_GT(refinement.indexes_mapping_size(), 0)
+          << "Refinement " << i << " has no indexes mapping";
+      RET_CHECK(refinement.has_z_refinement())
+          << "Refinement " << i << " has no Z refinement specified";
+      RET_CHECK(refinement.z_refinement().has_none() ^
+                refinement.z_refinement().has_copy() ^
+                refinement.z_refinement().has_assign_average())
+          << "Exactly one Z refinement should be specified";
+
+      const auto z_refinement = refinement.z_refinement();
+      if (z_refinement.has_assign_average()) {
+        RET_CHECK_GT(z_refinement.assign_average().indexes_for_average_size(),
+                     0)
+            << "When using assign average Z refinement at least one index for "
+               "averagin should be specified";
+      }
+    }
+
+    // Validate indexes mapping and get total number of refined landmarks.
+    ASSIGN_OR_RETURN(n_refined_landmarks_,
+                     GetNumberOfRefinedLandmarks(options_.refinement()));
+
+    // Validate that number of refinements and landmark streams is the same.
+    RET_CHECK_EQ(kLandmarks(cc).Count(), options_.refinement_size())
+        << "There are " << options_.refinement_size() << " refinements while "
+        << kLandmarks(cc).Count() << " landmark streams";
+
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // If any of the refinement landmarks is missing - refinement won't happen.
+    for (const auto& landmarks_stream : kLandmarks(cc)) {
+      if (landmarks_stream.IsEmpty()) {
+        return absl::OkStatus();
+      }
+    }
+
+    // Initialize refined landmarks list.
+    auto refined_landmarks = absl::make_unique<NormalizedLandmarkList>();
+    for (int i = 0; i < n_refined_landmarks_; ++i) {
+      refined_landmarks->add_landmark();
+    }
+
+    // Apply input landmarks to outpu refined landmarks in provided order.
+    for (int i = 0; i < kLandmarks(cc).Count(); ++i) {
+      const auto& landmarks = kLandmarks(cc)[i].Get();
+      const auto& refinement = options_.refinement(i);
+
+      // Check number of landmarks in mapping and stream are the same.
+      RET_CHECK_EQ(landmarks.landmark_size(), refinement.indexes_mapping_size())
+          << "There are " << landmarks.landmark_size()
+          << " refinement landmarks while mapping has "
+          << refinement.indexes_mapping_size();
+
+      // Refine X and Y.
+      RefineXY(refinement.indexes_mapping(), landmarks,
+               refined_landmarks.get());
+
+      // Refine Z.
+      RefineZ(refinement.indexes_mapping(), refinement.z_refinement(),
+              landmarks, refined_landmarks.get());
+
+      // Visibility and presence are not currently refined and are left as `0`.
+    }
+
+    kRefinedLandmarks(cc).Send(std::move(refined_landmarks));
+    return absl::OkStatus();
+  }
+
+ private:
+  LandmarksRefinementCalculatorOptions options_;
+  int n_refined_landmarks_ = 0;
+};
+
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksRefinementCalculatorImpl);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_refinement_calculator.h

@@ -0,0 +1,85 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+
+namespace api2 {
+
+// A calculator to refine one set of landmarks with another.
+//
+// Inputs:
+//   LANDMARKS: Multiple NormalizedLandmarkList to use for
+//     refinement. They will be applied to the resulting REFINED_LANDMARKS in
+//     the provided order. Each list should be non empty and contain the same
+//     amount of landmarks as indexes in mapping. Number of lists should be the
+//     same as number of refinements in options.
+//
+// Outputs:
+//   REFINED_LANDMARKS: A NormalizedLandmarkList with refined landmarks. Number
+//     of produced landmarks is equal to to the maximum index mapping number in
+//     calculator options (calculator verifies that there are no gaps in the
+//     mapping).
+//
+// Examples config:
+//   node {
+//     calculator: "LandmarksRefinementCalculator"
+//     input_stream: "LANDMARKS:0:mesh_landmarks"
+//     input_stream: "LANDMARKS:1:lips_landmarks"
+//     input_stream: "LANDMARKS:2:left_eye_landmarks"
+//     input_stream: "LANDMARKS:3:right_eye_landmarks"
+//     output_stream: "REFINED_LANDMARKS:landmarks"
+//     options: {
+//       [mediapipe.LandmarksRefinementCalculatorOptions.ext] {
+//         refinement: {
+//           indexes_mapping: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+//           z_refinement: { copy {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [0, 1, 2, 3]
+//           z_refinement: { none {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [4, 5]
+//           z_refinement: { none {} }
+//         }
+//         refinement: {
+//           indexes_mapping: [6, 7]
+//           z_refinement: { none {} }
+//         }
+//       }
+//     }
+//   }
+//
+class LandmarksRefinementCalculator : public NodeIntf {
+ public:
+  static constexpr Input<::mediapipe::NormalizedLandmarkList>::Multiple
+      kLandmarks{"LANDMARKS"};
+  static constexpr Output<::mediapipe::NormalizedLandmarkList>
+      kRefinedLandmarks{"REFINED_LANDMARKS"};
+
+  MEDIAPIPE_NODE_INTERFACE(LandmarksRefinementCalculator, kLandmarks,
+                           kRefinedLandmarks);
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_REFINEMENT_CALCULATOR_H_

mediapipe/calculators/util/landmarks_refinement_calculator.proto

@@ -0,0 +1,71 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message LandmarksRefinementCalculatorOptions {
+  extend CalculatorOptions {
+    optional LandmarksRefinementCalculatorOptions ext = 381914658;
+  }
+
+  // Do nothing and keep those Z that are already present in the resulting set
+  // of landmarks.
+  message ZRefinementNone {}
+
+  // Simply copy Z values from the given set of landmarks to the resulting set
+  // of landmarks.
+  message ZRefinementCopy {}
+
+  // Calculate average of the specified set of landmarks in the resulting set
+  // and use it as Z for all given landmarks when assigning their values to the
+  // resulting set of landmarks.
+  message ZRefinementAssignAverage {
+    // Indexes of the resulting landmarks to use for average. Should be non
+    // empty.
+    repeated int32 indexes_for_average = 1;
+  }
+
+  // Specifies the set of instructions on assigning z value from the given set
+  // of landmarks to the resulting set of landmarks.
+  message ZRefinement {
+    // Exactly one Z refinement option should be specified.
+    oneof z_refinement_options {
+      ZRefinementNone none = 1;
+      ZRefinementCopy copy = 2;
+      ZRefinementAssignAverage assign_average = 3;
+    }
+  }
+
+  // Specifies the set of instructions of assigning values to the resulting set
+  // of landmarks.
+  message Refinement {
+    // Maps indexes of the given set of landmarks to indexes of the resulting
+    // set of landmarks. Should be non empty and contain the same amount of
+    // indexes as landmarks in the corresponding input stream.
+    repeated int32 indexes_mapping = 1;
+
+    // Z refinement instructions.
+    optional ZRefinement z_refinement = 2;
+  }
+
+  // Refinement instructions for every landmarks input stream. Applied in the
+  // same order as defined. Should be the same amount of refinements as landmark
+  // input streams in the calculator. Union of index mappings should start with
+  // 0 and cover a contineous range.
+  repeated Refinement refinement = 1;
+}

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -86,11 +86,11 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
 }
 
 template <class LandmarkType>
-bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark,
-                                  bool utilize_visibility,
-                                  float visibility_threshold,
-                                  bool utilize_presence,
-                                  float presence_threshold) {
+bool IsLandmarkVisibleAndPresent(const LandmarkType& landmark,
+                                 bool utilize_visibility,
+                                 float visibility_threshold,
+                                 bool utilize_presence,
+                                 float presence_threshold) {
   if (utilize_visibility && landmark.has_visibility() &&
       landmark.visibility() < visibility_threshold) {
     return false;
@@ -153,12 +153,16 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
                              const Color& max_depth_line_color,
                              RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
+    if (landmark_connections[i] >= landmarks.landmark_size() ||
+        landmark_connections[i + 1] >= landmarks.landmark_size()) {
+      continue;
+    }
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
+    if (!IsLandmarkVisibleAndPresent<LandmarkType>(
             ld0, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold) ||
-        !IsLandmarkVisibileAndPresent<LandmarkType>(
+        !IsLandmarkVisibleAndPresent<LandmarkType>(
             ld1, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold)) {
       continue;
@@ -196,12 +200,16 @@ void AddConnections(const LandmarkListType& landmarks,
                     const Color& connection_color, float thickness,
                     bool normalized, RenderData* render_data) {
   for (int i = 0; i < landmark_connections.size(); i += 2) {
+    if (landmark_connections[i] >= landmarks.landmark_size() ||
+        landmark_connections[i + 1] >= landmarks.landmark_size()) {
+      continue;
+    }
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (!IsLandmarkVisibileAndPresent<LandmarkType>(
+    if (!IsLandmarkVisibleAndPresent<LandmarkType>(
             ld0, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold) ||
-        !IsLandmarkVisibileAndPresent<LandmarkType>(
+        !IsLandmarkVisibleAndPresent<LandmarkType>(
             ld1, utilize_visibility, visibility_threshold, utilize_presence,
             presence_threshold)) {
       continue;
@@ -317,7 +325,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const Landmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibileAndPresent<Landmark>(
+      if (!IsLandmarkVisibleAndPresent<Landmark>(
               landmark, options_.utilize_visibility(),
               options_.visibility_threshold(), options_.utilize_presence(),
               options_.presence_threshold())) {
@@ -363,7 +371,7 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) {
     for (int i = 0; i < landmarks.landmark_size(); ++i) {
       const NormalizedLandmark& landmark = landmarks.landmark(i);
 
-      if (!IsLandmarkVisibileAndPresent<NormalizedLandmark>(
+      if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
               landmark, options_.utilize_visibility(),
               options_.visibility_threshold(), options_.utilize_presence(),
               options_.presence_threshold())) {

mediapipe/calculators/util/rect_transformation_calculator.cc

@@ -36,7 +36,7 @@ inline float NormalizeRadians(float angle) {
 }  // namespace
 
 // Performs geometric transformation to the input Rect or NormalizedRect,
-// correpsonding to input stream RECT or NORM_RECT respectively. When the input
+// corresponding to input stream RECT or NORM_RECT respectively. When the input
 // is NORM_RECT, an addition input stream IMAGE_SIZE is required, which is a
 // std::pair<int, int> representing the image width and height.
 //