diff --git a/mediapipe/tasks/cc/components/containers/BUILD b/mediapipe/tasks/cc/components/containers/BUILD index b5c9427a..5004383d 100644 --- a/mediapipe/tasks/cc/components/containers/BUILD +++ b/mediapipe/tasks/cc/components/containers/BUILD @@ -30,15 +30,6 @@ cc_library( ], ) -cc_library( - name = "hand_landmarks_detection_result", - hdrs = ["hand_landmarks_detection_result.h"], - deps = [ - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - ], -) - cc_library( name = "category", srcs = ["category.cc"], diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD index 08493428..5c5073fc 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD @@ -110,12 +110,22 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "hand_landmarker_result", + hdrs = ["hand_landmarker_result.h"], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + ], +) + cc_library( name = "hand_landmarker", srcs = ["hand_landmarker.cc"], hdrs = ["hand_landmarker.h"], deps = [ ":hand_landmarker_graph", + ":hand_landmarker_result", "//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:port", "//mediapipe/framework/formats:classification_cc_proto", @@ -124,7 +134,6 @@ cc_library( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc/components:image_preprocessing", - "//mediapipe/tasks/cc/components/containers:hand_landmarks_detection_result", "//mediapipe/tasks/cc/components/processors:classifier_options", "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", "//mediapipe/tasks/cc/core:base_options", diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc index ec9790c3..3a9ed5bc 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc @@ -22,7 +22,6 @@ limitations under the License. #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/tasks/cc/common.h" -#include "mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h" #include "mediapipe/tasks/cc/components/image_preprocessing.h" #include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" #include "mediapipe/tasks/cc/core/base_task_api.h" @@ -34,6 +33,7 @@ limitations under the License. #include "mediapipe/tasks/cc/vision/core/image_processing_options.h" #include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h" #include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h" #include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" @@ -47,8 +47,6 @@ namespace { using HandLandmarkerGraphOptionsProto = ::mediapipe::tasks::vision:: hand_landmarker::proto::HandLandmarkerGraphOptions; -using ::mediapipe::tasks::components::containers::HandLandmarksDetectionResult; - constexpr char kHandLandmarkerGraphTypeName[] = "mediapipe.tasks.vision.hand_landmarker.HandLandmarkerGraph"; @@ -145,7 +143,7 @@ absl::StatusOr> HandLandmarker::Create( Packet empty_packet = status_or_packets.value()[kHandLandmarksStreamName]; result_callback( - {HandLandmarksDetectionResult()}, image_packet.Get(), + {HandLandmarkerResult()}, image_packet.Get(), empty_packet.Timestamp().Value() / kMicroSecondsPerMilliSecond); return; } @@ -173,7 +171,7 @@ absl::StatusOr> HandLandmarker::Create( std::move(packets_callback)); } -absl::StatusOr HandLandmarker::Detect( +absl::StatusOr HandLandmarker::Detect( mediapipe::Image image, std::optional image_processing_options) { if (image.UsesGpu()) { @@ -192,7 +190,7 @@ absl::StatusOr HandLandmarker::Detect( {kNormRectStreamName, MakePacket(std::move(norm_rect))}})); if (output_packets[kHandLandmarksStreamName].IsEmpty()) { - return {HandLandmarksDetectionResult()}; + return {HandLandmarkerResult()}; } return {{/* handedness= */ {output_packets[kHandednessStreamName] @@ -205,7 +203,7 @@ absl::StatusOr HandLandmarker::Detect( .Get>()}}}; } -absl::StatusOr HandLandmarker::DetectForVideo( +absl::StatusOr HandLandmarker::DetectForVideo( mediapipe::Image image, int64 timestamp_ms, std::optional image_processing_options) { if (image.UsesGpu()) { @@ -227,7 +225,7 @@ absl::StatusOr HandLandmarker::DetectForVideo( MakePacket(std::move(norm_rect)) .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}})); if (output_packets[kHandLandmarksStreamName].IsEmpty()) { - return {HandLandmarksDetectionResult()}; + return {HandLandmarkerResult()}; } return { {/* handedness= */ diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.h b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.h index 3538ab3f..6f96fc68 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.h +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.h @@ -24,12 +24,12 @@ limitations under the License. #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h" #include "mediapipe/tasks/cc/components/processors/classifier_options.h" #include "mediapipe/tasks/cc/core/base_options.h" #include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h" #include "mediapipe/tasks/cc/vision/core/image_processing_options.h" #include "mediapipe/tasks/cc/vision/core/running_mode.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h" namespace mediapipe { namespace tasks { @@ -70,9 +70,7 @@ struct HandLandmarkerOptions { // The user-defined result callback for processing live stream data. // The result callback should only be specified when the running mode is set // to RunningMode::LIVE_STREAM. - std::function, - const Image&, int64)> + std::function, const Image&, int64)> result_callback = nullptr; }; @@ -92,7 +90,7 @@ struct HandLandmarkerOptions { // 'y_center', 'width' and 'height' fields is NOT supported and will // result in an invalid argument error being returned. // Outputs: -// HandLandmarksDetectionResult +// HandLandmarkerResult // - The hand landmarks detection results. class HandLandmarker : tasks::vision::core::BaseVisionTaskApi { public: @@ -129,7 +127,7 @@ class HandLandmarker : tasks::vision::core::BaseVisionTaskApi { // The image can be of any size with format RGB or RGBA. // TODO: Describes how the input image will be preprocessed // after the yuv support is implemented. - absl::StatusOr Detect( + absl::StatusOr Detect( Image image, std::optional image_processing_options = std::nullopt); @@ -147,10 +145,10 @@ class HandLandmarker : tasks::vision::core::BaseVisionTaskApi { // The image can be of any size with format RGB or RGBA. It's required to // provide the video frame's timestamp (in milliseconds). The input timestamps // must be monotonically increasing. - absl::StatusOr - DetectForVideo(Image image, int64 timestamp_ms, - std::optional - image_processing_options = std::nullopt); + absl::StatusOr DetectForVideo( + Image image, int64 timestamp_ms, + std::optional image_processing_options = + std::nullopt); // Sends live image data to perform hand landmarks detection, and the results // will be available via the "result_callback" provided in the @@ -169,7 +167,7 @@ class HandLandmarker : tasks::vision::core::BaseVisionTaskApi { // invalid argument error being returned. // // The "result_callback" provides - // - A vector of HandLandmarksDetectionResult, each is the detected results + // - A vector of HandLandmarkerResult, each is the detected results // for a input frame. // - The const reference to the corresponding input image that the hand // landmarker runs on. Note that the const reference to the image will no diff --git a/mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h similarity index 77% rename from mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h rename to mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h index b341cd8d..5e51c244 100644 --- a/mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h @@ -13,20 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_HAND_LANDMARKS_DETECTION_RESULT_H_ -#define MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_HAND_LANDMARKS_DETECTION_RESULT_H_ +#ifndef MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ +#define MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" namespace mediapipe { namespace tasks { -namespace components { -namespace containers { +namespace vision { +namespace hand_landmarker { // The hand landmarks detection result from HandLandmarker, where each vector // element represents a single hand detected in the image. -struct HandLandmarksDetectionResult { +struct HandLandmarkerResult { // Classification of handedness. std::vector handedness; // Detected hand landmarks in normalized image coordinates. @@ -35,9 +35,9 @@ struct HandLandmarksDetectionResult { std::vector hand_world_landmarks; }; -} // namespace containers -} // namespace components +} // namespace hand_landmarker +} // namespace vision } // namespace tasks } // namespace mediapipe -#endif // MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_HAND_LANDMARKS_DETECTION_RESULT_H_ +#endif // MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc index ee8c3c10..fa49a4c1 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc @@ -32,12 +32,12 @@ limitations under the License. #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/tasks/cc/common.h" -#include "mediapipe/tasks/cc/components/containers/hand_landmarks_detection_result.h" #include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" #include "mediapipe/tasks/cc/components/containers/rect.h" #include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" #include "mediapipe/tasks/cc/core/base_options.h" #include "mediapipe/tasks/cc/vision/core/image_processing_options.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h" #include "mediapipe/tasks/cc/vision/utils/image_utils.h" #include "tensorflow/lite/core/shims/cc/shims_test_util.h" @@ -50,7 +50,6 @@ namespace { using ::file::Defaults; using ::mediapipe::file::JoinPath; -using ::mediapipe::tasks::components::containers::HandLandmarksDetectionResult; using ::mediapipe::tasks::components::containers::Rect; using ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; using ::mediapipe::tasks::vision::core::ImageProcessingOptions; @@ -95,9 +94,9 @@ LandmarksDetectionResult GetLandmarksDetectionResult( return result; } -HandLandmarksDetectionResult GetExpectedHandLandmarksDetectionResult( +HandLandmarkerResult GetExpectedHandLandmarkerResult( const std::vector& landmarks_file_names) { - HandLandmarksDetectionResult expected_results; + HandLandmarkerResult expected_results; for (const auto& file_name : landmarks_file_names) { const auto landmarks_detection_result = GetLandmarksDetectionResult(file_name); @@ -109,9 +108,9 @@ HandLandmarksDetectionResult GetExpectedHandLandmarksDetectionResult( return expected_results; } -void ExpectHandLandmarksDetectionResultsCorrect( - const HandLandmarksDetectionResult& actual_results, - const HandLandmarksDetectionResult& expected_results) { +void ExpectHandLandmarkerResultsCorrect( + const HandLandmarkerResult& actual_results, + const HandLandmarkerResult& expected_results) { const auto& actual_landmarks = actual_results.hand_landmarks; const auto& actual_handedness = actual_results.handedness; @@ -145,7 +144,7 @@ struct TestParams { // clockwise. int rotation; // Expected results from the hand landmarker model output. - HandLandmarksDetectionResult expected_results; + HandLandmarkerResult expected_results; }; class ImageModeTest : public testing::TestWithParam {}; @@ -213,7 +212,7 @@ TEST_P(ImageModeTest, Succeeds) { MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr hand_landmarker, HandLandmarker::Create(std::move(options))); - HandLandmarksDetectionResult hand_landmarker_results; + HandLandmarkerResult hand_landmarker_results; if (GetParam().rotation != 0) { ImageProcessingOptions image_processing_options; image_processing_options.rotation_degrees = GetParam().rotation; @@ -224,8 +223,8 @@ TEST_P(ImageModeTest, Succeeds) { MP_ASSERT_OK_AND_ASSIGN(hand_landmarker_results, hand_landmarker->Detect(image)); } - ExpectHandLandmarksDetectionResultsCorrect(hand_landmarker_results, - GetParam().expected_results); + ExpectHandLandmarkerResultsCorrect(hand_landmarker_results, + GetParam().expected_results); MP_ASSERT_OK(hand_landmarker->Close()); } @@ -237,8 +236,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kThumbUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kThumbUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUp", @@ -246,8 +244,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kPointingUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kPointingUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUpRotated", @@ -255,7 +252,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ -90, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( + GetExpectedHandLandmarkerResult( {kPointingUpRotatedLandmarksFilename}), }, TestParams{ @@ -315,7 +312,7 @@ TEST_P(VideoModeTest, Succeeds) { HandLandmarker::Create(std::move(options))); const auto expected_results = GetParam().expected_results; for (int i = 0; i < iterations; ++i) { - HandLandmarksDetectionResult hand_landmarker_results; + HandLandmarkerResult hand_landmarker_results; if (GetParam().rotation != 0) { ImageProcessingOptions image_processing_options; image_processing_options.rotation_degrees = GetParam().rotation; @@ -326,8 +323,8 @@ TEST_P(VideoModeTest, Succeeds) { MP_ASSERT_OK_AND_ASSIGN(hand_landmarker_results, hand_landmarker->DetectForVideo(image, i)); } - ExpectHandLandmarksDetectionResultsCorrect(hand_landmarker_results, - expected_results); + ExpectHandLandmarkerResultsCorrect(hand_landmarker_results, + expected_results); } MP_ASSERT_OK(hand_landmarker->Close()); } @@ -340,8 +337,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kThumbUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kThumbUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUp", @@ -349,8 +345,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kPointingUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kPointingUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUpRotated", @@ -358,7 +353,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ -90, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( + GetExpectedHandLandmarkerResult( {kPointingUpRotatedLandmarksFilename}), }, TestParams{ @@ -383,9 +378,8 @@ TEST_F(LiveStreamModeTest, FailsWithCallingWrongMethod) { options->base_options.model_asset_path = JoinPath("./", kTestDataDirectory, kHandLandmarkerBundleAsset); options->running_mode = core::RunningMode::LIVE_STREAM; - options->result_callback = - [](absl::StatusOr results, - const Image& image, int64 timestamp_ms) {}; + options->result_callback = [](absl::StatusOr results, + const Image& image, int64 timestamp_ms) {}; MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr hand_landmarker, HandLandmarker::Create(std::move(options))); @@ -416,23 +410,23 @@ TEST_P(LiveStreamModeTest, Succeeds) { options->base_options.model_asset_path = JoinPath("./", kTestDataDirectory, GetParam().test_model_file); options->running_mode = core::RunningMode::LIVE_STREAM; - std::vector hand_landmarker_results; + std::vector hand_landmarker_results; std::vector> image_sizes; std::vector timestamps; - options->result_callback = - [&hand_landmarker_results, &image_sizes, ×tamps]( - absl::StatusOr results, - const Image& image, int64 timestamp_ms) { - MP_ASSERT_OK(results.status()); - hand_landmarker_results.push_back(std::move(results.value())); - image_sizes.push_back({image.width(), image.height()}); - timestamps.push_back(timestamp_ms); - }; + options->result_callback = [&hand_landmarker_results, &image_sizes, + ×tamps]( + absl::StatusOr results, + const Image& image, int64 timestamp_ms) { + MP_ASSERT_OK(results.status()); + hand_landmarker_results.push_back(std::move(results.value())); + image_sizes.push_back({image.width(), image.height()}); + timestamps.push_back(timestamp_ms); + }; MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr hand_landmarker, HandLandmarker::Create(std::move(options))); for (int i = 0; i < iterations; ++i) { - HandLandmarksDetectionResult hand_landmarker_results; + HandLandmarkerResult hand_landmarker_results; if (GetParam().rotation != 0) { ImageProcessingOptions image_processing_options; image_processing_options.rotation_degrees = GetParam().rotation; @@ -450,8 +444,8 @@ TEST_P(LiveStreamModeTest, Succeeds) { const auto expected_results = GetParam().expected_results; for (int i = 0; i < hand_landmarker_results.size(); ++i) { - ExpectHandLandmarksDetectionResultsCorrect(hand_landmarker_results[i], - expected_results); + ExpectHandLandmarkerResultsCorrect(hand_landmarker_results[i], + expected_results); } for (const auto& image_size : image_sizes) { EXPECT_EQ(image_size.first, image.width()); @@ -472,8 +466,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kThumbUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kThumbUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUp", @@ -481,8 +474,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ 0, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( - {kPointingUpLandmarksFilename}), + GetExpectedHandLandmarkerResult({kPointingUpLandmarksFilename}), }, TestParams{ /* test_name= */ "LandmarksPointingUpRotated", @@ -490,7 +482,7 @@ INSTANTIATE_TEST_SUITE_P( /* test_model_file= */ kHandLandmarkerBundleAsset, /* rotation= */ -90, /* expected_results = */ - GetExpectedHandLandmarksDetectionResult( + GetExpectedHandLandmarkerResult( {kPointingUpRotatedLandmarksFilename}), }, TestParams{