From f9f6acffed8b58ebed637643ff67bda8a892e4ef Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 30 Jan 2023 09:15:06 -0800 Subject: [PATCH] Make NORM_RECT optional for GestureRecognizerGraph and add PALM_DETECTION output PORT PiperOrigin-RevId: 505712542 --- .../tasks/cc/vision/gesture_recognizer/BUILD | 1 + .../gesture_recognizer_graph.cc | 41 +++++++++++++++---- .../hand_detector/hand_detector_graph.cc | 17 ++++---- .../hand_landmarker/hand_landmarker_graph.cc | 16 ++++---- .../hand_landmarks_detector_graph.cc | 11 ++--- 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD index d473a8dc..7ffae6ff 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD @@ -140,6 +140,7 @@ cc_library( "//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:port", "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:rect_cc_proto", diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc index 2d949c41..b6f6c88d 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc @@ -22,6 +22,7 @@ limitations under the License. #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -68,6 +69,9 @@ constexpr char kHandednessTag[] = "HANDEDNESS"; constexpr char kImageSizeTag[] = "IMAGE_SIZE"; constexpr char kHandGesturesTag[] = "HAND_GESTURES"; constexpr char kHandTrackingIdsTag[] = "HAND_TRACKING_IDS"; +constexpr char kRectNextFrameTag[] = "HAND_RECT_NEXT_FRAME"; +constexpr char kPalmRectsTag[] = "PALM_RECTS"; +constexpr char kPalmDetectionsTag[] = "PALM_DETECTIONS"; constexpr char kHandLandmarkerBundleAssetName[] = "hand_landmarker.task"; constexpr char kHandGestureRecognizerBundleAssetName[] = "hand_gesture_recognizer.task"; @@ -77,6 +81,9 @@ struct GestureRecognizerOutputs { Source> handedness; Source> hand_landmarks; Source> hand_world_landmarks; + Source> hand_rects_next_frame; + Source> palm_rects; + Source> palm_detections; Source image; }; @@ -135,9 +142,10 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, // Inputs: // IMAGE - Image // Image to perform hand gesture recognition on. -// NORM_RECT - NormalizedRect +// NORM_RECT - NormalizedRect @Optional // Describes image rotation and region of image to perform landmarks -// detection on. +// detection on. If not provided, whole image is used for gesture +// recognition. // // Outputs: // HAND_GESTURES - std::vector @@ -208,11 +216,12 @@ class GestureRecognizerGraph : public core::ModelTaskGraph { !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) .IsAvailable())); } - ASSIGN_OR_RETURN(auto hand_gesture_recognition_output, - BuildGestureRecognizerGraph( - *sc->MutableOptions(), - graph[Input(kImageTag)], - graph[Input(kNormRectTag)], graph)); + ASSIGN_OR_RETURN( + auto hand_gesture_recognition_output, + BuildGestureRecognizerGraph( + *sc->MutableOptions(), + graph[Input(kImageTag)], + graph[Input::Optional(kNormRectTag)], graph)); hand_gesture_recognition_output.gesture >> graph[Output>(kHandGesturesTag)]; hand_gesture_recognition_output.handedness >> @@ -222,6 +231,12 @@ class GestureRecognizerGraph : public core::ModelTaskGraph { hand_gesture_recognition_output.hand_world_landmarks >> graph[Output>(kWorldLandmarksTag)]; hand_gesture_recognition_output.image >> graph[Output(kImageTag)]; + hand_gesture_recognition_output.hand_rects_next_frame >> + graph[Output>(kRectNextFrameTag)]; + hand_gesture_recognition_output.palm_rects >> + graph[Output>(kPalmRectsTag)]; + hand_gesture_recognition_output.palm_detections >> + graph[Output>(kPalmDetectionsTag)]; return graph.GetConfig(); } @@ -279,7 +294,17 @@ class GestureRecognizerGraph : public core::ModelTaskGraph { /*handedness=*/handedness, /*hand_landmarks=*/hand_landmarks, /*hand_world_landmarks=*/hand_world_landmarks, - /*image=*/hand_landmarker_graph[Output(kImageTag)]}; + /*hand_rects_next_frame =*/ + hand_landmarker_graph[Output>( + kRectNextFrameTag)], + /*palm_rects =*/ + hand_landmarker_graph[Output>( + kPalmRectsTag)], + /*palm_detections =*/ + hand_landmarker_graph[Output>( + kPalmDetectionsTag)], + /*image=*/hand_landmarker_graph[Output(kImageTag)], + }; } }; diff --git a/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc b/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc index 49958e36..d7163e33 100644 --- a/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc @@ -150,9 +150,9 @@ void ConfigureRectTransformationCalculator( // Inputs: // IMAGE - Image // Image to perform detection on. -// NORM_RECT - NormalizedRect -// Describes image rotation and region of image to perform detection -// on. +// NORM_RECT - NormalizedRect @Optional +// Describes image rotation and region of image to perform detection on. If +// not provided, whole image is used for hand detection. // // Outputs: // PALM_DETECTIONS - std::vector @@ -197,11 +197,12 @@ class HandDetectorGraph : public core::ModelTaskGraph { ASSIGN_OR_RETURN(const auto* model_resources, CreateModelResources(sc)); Graph graph; - ASSIGN_OR_RETURN(auto hand_detection_outs, - BuildHandDetectionSubgraph( - sc->Options(), - *model_resources, graph[Input(kImageTag)], - graph[Input(kNormRectTag)], graph)); + ASSIGN_OR_RETURN( + auto hand_detection_outs, + BuildHandDetectionSubgraph( + sc->Options(), *model_resources, + graph[Input(kImageTag)], + graph[Input::Optional(kNormRectTag)], graph)); hand_detection_outs.palm_detections >> graph[Output>(kPalmDetectionsTag)]; hand_detection_outs.hand_rects >> diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc index 05ad97ef..74d288ac 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc @@ -136,9 +136,10 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, // Inputs: // IMAGE - Image // Image to perform hand landmarks detection on. -// NORM_RECT - NormalizedRect +// NORM_RECT - NormalizedRect @Optional // Describes image rotation and region of image to perform landmarks -// detection on. +// detection on. If not provided, whole image is used for hand landmarks +// detection. // // Outputs: // LANDMARKS: - std::vector @@ -218,11 +219,12 @@ class HandLandmarkerGraph : public core::ModelTaskGraph { !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) .IsAvailable())); } - ASSIGN_OR_RETURN(auto hand_landmarker_outputs, - BuildHandLandmarkerGraph( - sc->Options(), - graph[Input(kImageTag)], - graph[Input(kNormRectTag)], graph)); + ASSIGN_OR_RETURN( + auto hand_landmarker_outputs, + BuildHandLandmarkerGraph( + sc->Options(), + graph[Input(kImageTag)], + graph[Input::Optional(kNormRectTag)], graph)); hand_landmarker_outputs.landmark_lists >> graph[Output>(kLandmarksTag)]; hand_landmarker_outputs.world_landmark_lists >> diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc index 4ea066aa..914bc30f 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc @@ -243,11 +243,12 @@ class SingleHandLandmarksDetectorGraph : public core::ModelTaskGraph { const auto* model_resources, CreateModelResources(sc)); Graph graph; - ASSIGN_OR_RETURN(auto hand_landmark_detection_outs, - BuildSingleHandLandmarksDetectorGraph( - sc->Options(), - *model_resources, graph[Input(kImageTag)], - graph[Input(kHandRectTag)], graph)); + ASSIGN_OR_RETURN( + auto hand_landmark_detection_outs, + BuildSingleHandLandmarksDetectorGraph( + sc->Options(), *model_resources, + graph[Input(kImageTag)], + graph[Input::Optional(kHandRectTag)], graph)); hand_landmark_detection_outs.hand_landmarks >> graph[Output(kLandmarksTag)]; hand_landmark_detection_outs.world_hand_landmarks >>