Project import generated by Copybara.

GitOrigin-RevId: 73d686c40057684f8bfaca285368bf1813f9fc26
2022-03-21 12:12:39 -07:00
parent e6c19885c6
commit cc6a2f7af6
266 changed files with 3658 additions and 1681 deletions
@@ -1,5 +1,5 @@
-# MediaPipe graph to detect faces. (CPU input, and inference is executed on
-# CPU.)
+# MediaPipe graph to detect faces. (GPU input, and inference is executed on
+# GPU.)
 #
 # It is required that "face_detection_short_range.tflite" is available at
 # "mediapipe/modules/face_detection/face_detection_short_range.tflite"
@@ -1,5 +1,5 @@
-# MediaPipe graph to detect faces. (CPU input, and inference is executed on
-# CPU.)
+# MediaPipe graph to detect faces. (GPU input, and inference is executed on
+# GPU.)
 #
 # It is required that "face_detection_short_range.tflite" is available at
 # "mediapipe/modules/face_detection/face_detection_short_range.tflite"
@@ -164,7 +164,6 @@ mediapipe_simple_subgraph(
    graph = "hand_landmark_landmarks_to_roi.pbtxt",
    register_as = "HandLandmarkLandmarksToRoi",
    deps = [
-        "//mediapipe/calculators/core:split_landmarks_calculator",
        "//mediapipe/calculators/util:rect_transformation_calculator",
        "//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
    ],
@@ -24,13 +24,21 @@ namespace mediapipe {

 namespace {

+// NORM_LANDMARKS is either the full set of landmarks for the hand, or
+// a subset of the hand landmarks (indices 0, 1, 2, 3, 5, 6, 9, 10, 13, 14,
+// 17 and 18). The latter is the legacy behavior, please just pass in
+// the full set of hand landmarks.
+//
+// TODO: update clients to just pass all the landmarks in.
 constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
 constexpr char kNormRectTag[] = "NORM_RECT";
 constexpr char kImageSizeTag[] = "IMAGE_SIZE";
+// Indices within the partial landmarks.
 constexpr int kWristJoint = 0;
 constexpr int kMiddleFingerPIPJoint = 6;
 constexpr int kIndexFingerPIPJoint = 4;
 constexpr int kRingFingerPIPJoint = 8;
+constexpr int kNumLandmarks = 21;
 constexpr float kTargetAngle = M_PI * 0.5f;

 inline float NormalizeRadians(float angle) {
@@ -150,8 +158,7 @@ class HandLandmarksToRectCalculator : public CalculatorBase {

    std::pair<int, int> image_size =
        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
-    const auto& landmarks =
-        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
+    const auto landmarks = GetPartialLandmarks(cc);
    auto output_rect = absl::make_unique<NormalizedRect>();
    MP_RETURN_IF_ERROR(
        NormalizedLandmarkListToRect(landmarks, image_size, output_rect.get()));
@@ -161,6 +168,25 @@ class HandLandmarksToRectCalculator : public CalculatorBase {

    return absl::OkStatus();
  }
+
+ private:
+  NormalizedLandmarkList GetPartialLandmarks(CalculatorContext* cc) {
+    const auto& landmarks =
+        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
+    if (landmarks.landmark_size() == kNumLandmarks) {
+      static constexpr int kPartialLandmarkIndices[]{0, 1,  2,  3,  5,  6,
+                                                     9, 10, 13, 14, 17, 18};
+      NormalizedLandmarkList partial_landmarks;
+      for (int i : kPartialLandmarkIndices) {
+        *partial_landmarks.add_landmark() = landmarks.landmark(i);
+      }
+      return partial_landmarks;
+    } else {
+      // Assume the calculator is receiving the partial landmarks directly.
+      // This is the legacy behavior.
+      return landmarks;
+    }
+  }
 };
 REGISTER_CALCULATOR(HandLandmarksToRectCalculator);

@@ -11,35 +11,11 @@ input_stream: "IMAGE_SIZE:image_size"
 # ROI according to landmarks. (NormalizedRect)
 output_stream: "ROI:roi"

-# Extracts a subset of the hand landmarks that are relatively more stable across
-# frames (e.g. comparing to finger tips) for computing the bounding box. The box
-# will later be expanded to contain the entire hand. In this approach, it is
-# more robust to drastically changing hand size.
-# The landmarks extracted are: wrist, MCP/PIP of five fingers.
-node {
-  calculator: "SplitNormalizedLandmarkListCalculator"
-  input_stream: "landmarks"
-  output_stream: "partial_landmarks"
-  options: {
-    [mediapipe.SplitVectorCalculatorOptions.ext] {
-      ranges: { begin: 0 end: 4 }
-      ranges: { begin: 5 end: 7 }
-      ranges: { begin: 9 end: 11 }
-      ranges: { begin: 13 end: 15 }
-      ranges: { begin: 17 end: 19 }
-      combine_outputs: true
-    }
-  }
-}
-
 # Converts the hand landmarks into a rectangle (normalized by image size)
-# that encloses the hand. The calculator uses a subset of all hand landmarks
-# extracted from SplitNormalizedLandmarkListCalculator above to
-# calculate the bounding box and the rotation of the output rectangle. Please
-# see the comments in the calculator for more detail.
+# that encloses the hand.
 node {
  calculator: "HandLandmarksToRectCalculator"
-  input_stream: "NORM_LANDMARKS:partial_landmarks"
+  input_stream: "NORM_LANDMARKS:landmarks"
  input_stream: "IMAGE_SIZE:image_size"
  output_stream: "NORM_RECT:hand_rect_from_landmarks"
 }
@@ -31,7 +31,7 @@ mediapipe_simple_subgraph(
        ":face_detection_front_detections_to_roi",
        ":face_landmarks_from_pose_to_recrop_roi",
        ":face_tracking",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
        "//mediapipe/calculators/image:image_properties_calculator",
        "//mediapipe/modules/face_detection:face_detection_short_range_by_roi_gpu",
        "//mediapipe/modules/face_landmark:face_landmark_gpu",
@@ -46,7 +46,7 @@ mediapipe_simple_subgraph(
        ":face_detection_front_detections_to_roi",
        ":face_landmarks_from_pose_to_recrop_roi",
        ":face_tracking",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
        "//mediapipe/calculators/image:image_properties_calculator",
        "//mediapipe/modules/face_detection:face_detection_short_range_by_roi_cpu",
        "//mediapipe/modules/face_landmark:face_landmark_cpu",
@@ -131,7 +131,6 @@ mediapipe_simple_subgraph(
    graph = "hand_landmarks_to_roi.pbtxt",
    register_as = "HandLandmarksToRoi",
    deps = [
-        "//mediapipe/calculators/core:split_landmarks_calculator",
        "//mediapipe/calculators/util:rect_transformation_calculator",
        "//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
    ],
@@ -191,7 +190,7 @@ mediapipe_simple_subgraph(
    deps = [
        "//mediapipe/calculators/core:constant_side_packet_calculator",
        "//mediapipe/calculators/core:side_packet_to_stream_calculator",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
        "//mediapipe/calculators/util:set_landmark_visibility_calculator",
    ],
 )
@@ -202,7 +201,7 @@ mediapipe_simple_subgraph(
    register_as = "HandLandmarksLeftAndRightGpu",
    deps = [
        ":hand_landmarks_from_pose_gpu",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
    ],
 )

@@ -212,7 +211,7 @@ mediapipe_simple_subgraph(
    register_as = "HandLandmarksLeftAndRightCpu",
    deps = [
        ":hand_landmarks_from_pose_cpu",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
    ],
 )

@@ -232,7 +231,7 @@ mediapipe_simple_subgraph(
    graph = "hand_visibility_from_hand_landmarks_from_pose.pbtxt",
    register_as = "HandVisibilityFromHandLandmarksFromPose",
    deps = [
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
        "//mediapipe/calculators/util:landmark_visibility_calculator",
        "//mediapipe/calculators/util:thresholding_calculator",
    ],
@@ -10,31 +10,11 @@ input_stream: "IMAGE_SIZE:image_size"
 # ROI according to the hand landmarks. (NormalizedRect)
 output_stream: "ROI:roi"

-# Gets hand palm landmarks.
-node {
-  calculator: "SplitNormalizedLandmarkListCalculator"
-  input_stream: "hand_landmarks"
-  output_stream: "palm_landmarks"
-  options: {
-    [mediapipe.SplitVectorCalculatorOptions.ext] {
-      ranges: { begin: 0 end: 4 }
-      ranges: { begin: 5 end: 7 }
-      ranges: { begin: 9 end: 11 }
-      ranges: { begin: 13 end: 15 }
-      ranges: { begin: 17 end: 19 }
-      combine_outputs: true
-    }
-  }
-}
-
 # Converts the hand landmarks into a rectangle (normalized by image size)
-# that encloses the hand. The calculator uses a subset of all hand landmarks
-# extracted from SplitNormalizedLandmarkListCalculator above to
-# calculate the bounding box and the rotation of the output rectangle. Please
-# see the comments in the calculator for more detail.
+# that encloses the hand.
 node {
  calculator: "HandLandmarksToRectCalculator"
-  input_stream: "NORM_LANDMARKS:palm_landmarks"
+  input_stream: "NORM_LANDMARKS:hand_landmarks"
  input_stream: "IMAGE_SIZE:image_size"
  output_stream: "NORM_RECT:palm_landmarks_rect"
 }
@@ -382,6 +382,23 @@ cc_library(
    alwayslink = 1,
 )

+cc_test(
+    name = "filter_detection_calculator_test",
+    srcs = ["filter_detection_calculator_test.cc"],
+    deps = [
+        ":filter_detection_calculator",  # build_cleaner: keep
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/util:packet_test_util",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_test(
    name = "box_util_test",
    srcs = ["box_util_test.cc"],
@@ -42,33 +42,6 @@ using mediapipe::RE2;
 using Detections = std::vector<Detection>;
 using Strings = std::vector<std::string>;

-}  // namespace
-
-// Filters the entries in a Detection to only those with valid scores
-// for the specified allowed labels. Allowed labels are provided as a
-// vector<std::string> in an optional input side packet. Allowed labels can
-// contain simple strings or regular expressions. The valid score range
-// can be set in the options.The allowed labels can be provided as
-// vector<std::string> (LABELS) or CSV std::string (LABELS_CSV) containing class
-// names of allowed labels. Note: Providing an empty vector in the input side
-// packet Packet causes this calculator to act as a sink if
-// empty_allowed_labels_means_allow_everything is set to false (default value).
-// To allow all labels, use the calculator with no input side packet stream, or
-// set empty_allowed_labels_means_allow_everything to true.
-//
-// Example config:
-// node {
-//   calculator: "FilterDetectionCalculator"
-//   input_stream: "DETECTIONS:detections"
-//   output_stream: "DETECTIONS:filtered_detections"
-//   input_side_packet: "LABELS:allowed_labels"
-//   options: {
-//     [mediapipe.FilterDetectionCalculatorOptions.ext]: {
-//       min_score: 0.5
-//     }
-//   }
-// }
-
 struct FirstGreaterComparator {
  bool operator()(const std::pair<float, int>& a,
                  const std::pair<float, int>& b) const {
@@ -112,6 +85,33 @@ absl::Status SortLabelsByDecreasingScore(const Detection& detection,
  return absl::OkStatus();
 }

+}  // namespace
+
+// Filters the entries in a Detection to only those with valid scores
+// for the specified allowed labels. Allowed labels are provided as a
+// std::vector<std::string> in an optional input side packet. Allowed labels can
+// contain simple strings or regular expressions. The valid score range
+// can be set in the options.The allowed labels can be provided as
+// std::vector<std::string> (LABELS) or CSV string (LABELS_CSV) containing class
+// names of allowed labels. Note: Providing an empty vector in the input side
+// packet Packet causes this calculator to act as a sink if
+// empty_allowed_labels_means_allow_everything is set to false (default value).
+// To allow all labels, use the calculator with no input side packet stream, or
+// set empty_allowed_labels_means_allow_everything to true.
+//
+// Example config:
+// node {
+//   calculator: "FilterDetectionCalculator"
+//   input_stream: "DETECTIONS:detections"
+//   output_stream: "DETECTIONS:filtered_detections"
+//   input_side_packet: "LABELS:allowed_labels"
+//   options: {
+//     [mediapipe.FilterDetectionCalculatorOptions.ext]: {
+//       min_score: 0.5
+//     }
+//   }
+// }
+
 class FilterDetectionCalculator : public CalculatorBase {
 public:
  static absl::Status GetContract(CalculatorContract* cc);
@@ -196,7 +196,7 @@ absl::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
  if (cc->Inputs().HasTag(kDetectionsTag)) {
    detections = cc->Inputs().Tag(kDetectionsTag).Get<Detections>();
  } else if (cc->Inputs().HasTag(kDetectionTag)) {
-    detections.emplace_back(cc->Inputs().Tag(kDetectionsTag).Get<Detection>());
+    detections.emplace_back(cc->Inputs().Tag(kDetectionTag).Get<Detection>());
  }
  std::unique_ptr<Detections> outputs(new Detections);
  for (const auto& input : detections) {
@@ -229,7 +229,7 @@ absl::Status FilterDetectionCalculator::Process(CalculatorContext* cc) {
        .Add(outputs.release(), cc->InputTimestamp());
  } else if (!outputs->empty()) {
    cc->Outputs()
-        .Tag(kDetectionsTag)
+        .Tag(kDetectionTag)
        .Add(new Detection((*outputs)[0]), cc->InputTimestamp());
  }
  return absl::OkStatus();
@@ -0,0 +1,71 @@
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/util/packet_test_util.h"
+#include "testing/base/public/gmock.h"
+#include "testing/base/public/gunit.h"
+
+namespace mediapipe {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::EqualsProto;
+
+TEST(FilterDetectionCalculatorTest, DetectionFilterTest) {
+  auto runner = std::make_unique<CalculatorRunner>(
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"pb(
+        calculator: "FilterDetectionCalculator"
+        input_stream: "DETECTION:input"
+        output_stream: "DETECTION:output"
+        options {
+          [mediapipe.FilterDetectionCalculatorOptions.ext]: { min_score: 0.6 }
+        }
+      )pb"));
+
+  runner->MutableInputs()->Tag("DETECTION").packets = {
+      MakePacket<Detection>(ParseTextProtoOrDie<Detection>(R"pb(
+        label: "a"
+        label: "b"
+        label: "c"
+        score: 1
+        score: 0.8
+        score: 0.3
+      )pb"))
+          .At(Timestamp(20)),
+      MakePacket<Detection>(ParseTextProtoOrDie<Detection>(R"pb(
+        label: "a"
+        label: "b"
+        label: "c"
+        score: 0.6
+        score: 0.4
+        score: 0.2
+      )pb"))
+          .At(Timestamp(40)),
+  };
+
+  // Run graph.
+  MP_ASSERT_OK(runner->Run());
+
+  // Check output.
+  EXPECT_THAT(
+      runner->Outputs().Tag("DETECTION").packets,
+      ElementsAre(PacketContainsTimestampAndPayload<Detection>(
+                      Eq(Timestamp(20)),
+                      EqualsProto(R"pb(
+                        label: "a" label: "b" score: 1 score: 0.8
+                      )pb")),  // Packet 1 at timestamp 20.
+                  PacketContainsTimestampAndPayload<Detection>(
+                      Eq(Timestamp(40)),
+                      EqualsProto(R"pb(
+                        label: "a" score: 0.6
+                      )pb"))  // Packet 2 at timestamp 40.
+                  ));
+}
+
+}  // namespace
+}  // namespace mediapipe
@@ -67,7 +67,7 @@ mediapipe_simple_subgraph(
    register_as = "TensorsToPoseLandmarksAndSegmentation",
    deps = [
        "//mediapipe/calculators/core:gate_calculator",
-        "//mediapipe/calculators/core:split_landmarks_calculator",
+        "//mediapipe/calculators/core:split_proto_list_calculator",
        "//mediapipe/calculators/core:split_vector_calculator",
        "//mediapipe/calculators/tensor:tensors_to_floats_calculator",
        "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
@@ -2,9 +2,12 @@

 type: "SelfieSegmentationModelLoader"

-# An integer 0 or 1. Use 0 to select a general-purpose model (operating on a
-# 256x256 tensor), and 1 to select a model (operating on a 256x144 tensor) more
-# optimized for landscape images. If unspecified, functions as set to 0. (int)
+# model_selection is an integer.
+# Use 0 to select a general-purpose model (operating on a 256x256 tensor).
+# Use 1 to select a model (operating on a 256x144 tensor) more optimized for
+# landscape images.
+#
+# If unspecified, 0 is selected by default.
 input_side_packet: "MODEL_SELECTION:model_selection"

 # TF Lite model represented as a FlatBuffer.
@@ -39,6 +42,7 @@ node {
          }
        }
      }
+      #
    }
  }
 }