From 69d354fc89173035007280daef793b7a640542fe Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 23 Jan 2023 12:09:41 -0800 Subject: [PATCH] Use c++ struct as hand landmark detection results. PiperOrigin-RevId: 504048095 --- .../tasks/cc/components/containers/BUILD | 9 ++ .../containers/classification_result.cc | 13 +++ .../containers/classification_result.h | 7 ++ .../cc/components/containers/landmark.cc | 65 +++++++++++ .../tasks/cc/components/containers/landmark.h | 103 ++++++++++++++++++ .../tasks/cc/vision/hand_landmarker/BUILD | 3 + .../vision/hand_landmarker/hand_landmarker.cc | 60 ++++++---- .../hand_landmarker/hand_landmarker_result.cc | 56 ++++++++++ .../hand_landmarker/hand_landmarker_result.h | 15 ++- .../hand_landmarker_result_test.cc | 88 +++++++++++++++ .../hand_landmarker/hand_landmarker_test.cc | 62 ++++++++--- 11 files changed, 439 insertions(+), 42 deletions(-) create mode 100644 mediapipe/tasks/cc/components/containers/landmark.cc create mode 100644 mediapipe/tasks/cc/components/containers/landmark.h create mode 100644 mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.cc create mode 100644 mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result_test.cc diff --git a/mediapipe/tasks/cc/components/containers/BUILD b/mediapipe/tasks/cc/components/containers/BUILD index 0750a148..a7307b2c 100644 --- a/mediapipe/tasks/cc/components/containers/BUILD +++ b/mediapipe/tasks/cc/components/containers/BUILD @@ -62,3 +62,12 @@ cc_library( "//mediapipe/tasks/cc/components/containers/proto:embeddings_cc_proto", ], ) + +cc_library( + name = "landmark", + srcs = ["landmark.cc"], + hdrs = ["landmark.h"], + deps = [ + "//mediapipe/framework/formats:landmark_cc_proto", + ], +) diff --git a/mediapipe/tasks/cc/components/containers/classification_result.cc b/mediapipe/tasks/cc/components/containers/classification_result.cc index 98583ff1..f2d88406 100644 --- a/mediapipe/tasks/cc/components/containers/classification_result.cc +++ b/mediapipe/tasks/cc/components/containers/classification_result.cc @@ -40,6 +40,19 @@ Classifications ConvertToClassifications(const proto::Classifications& proto) { return classifications; } +Classifications ConvertToClassifications( + const mediapipe::ClassificationList& proto, int head_index, + std::optional head_name) { + Classifications classifications; + classifications.categories.reserve(proto.classification_size()); + for (const auto& classification : proto.classification()) { + classifications.categories.push_back(ConvertToCategory(classification)); + } + classifications.head_index = head_index; + classifications.head_name = head_name; + return classifications; +} + ClassificationResult ConvertToClassificationResult( const proto::ClassificationResult& proto) { ClassificationResult classification_result; diff --git a/mediapipe/tasks/cc/components/containers/classification_result.h b/mediapipe/tasks/cc/components/containers/classification_result.h index 88273fd0..e359fb33 100644 --- a/mediapipe/tasks/cc/components/containers/classification_result.h +++ b/mediapipe/tasks/cc/components/containers/classification_result.h @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/tasks/cc/components/containers/category.h" #include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" @@ -58,6 +59,12 @@ struct ClassificationResult { // Classifications struct. Classifications ConvertToClassifications(const proto::Classifications& proto); +// Utility function to convert from ClassificationList proto to +// Classifications struct. +Classifications ConvertToClassifications( + const mediapipe::ClassificationList& proto, int head_index = 0, + std::optional head_name = std::nullopt); + // Utility function to convert from ClassificationResult proto to // ClassificationResult struct. ClassificationResult ConvertToClassificationResult( diff --git a/mediapipe/tasks/cc/components/containers/landmark.cc b/mediapipe/tasks/cc/components/containers/landmark.cc new file mode 100644 index 00000000..6d80cb83 --- /dev/null +++ b/mediapipe/tasks/cc/components/containers/landmark.cc @@ -0,0 +1,65 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +#include +#include + +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe::tasks::components::containers { + +Landmark ConvertToLandmark(const mediapipe::Landmark& proto) { + return {/*x=*/proto.x(), /*y=*/proto.y(), /*z=*/proto.z(), + /*visibility=*/proto.has_visibility() + ? std::optional(proto.visibility()) + : std::nullopt, + /*presence=*/proto.has_presence() + ? std::optional(proto.presence()) + : std::nullopt}; +} + +NormalizedLandmark ConvertToNormalizedLandmark( + const mediapipe::NormalizedLandmark& proto) { + return {/*x=*/proto.x(), /*y=*/proto.y(), /*z=*/proto.z(), + /*visibility=*/proto.has_visibility() + ? std::optional(proto.visibility()) + : std::nullopt, + /*presence=*/proto.has_presence() + ? std::optional(proto.presence()) + : std::nullopt}; +} + +Landmarks ConvertToLandmarks(const mediapipe::LandmarkList& proto) { + Landmarks landmarks; + landmarks.landmarks.reserve(proto.landmark_size()); + for (const auto& landmark : proto.landmark()) { + landmarks.landmarks.push_back(ConvertToLandmark(landmark)); + } + return landmarks; +} + +NormalizedLandmarks ConvertToNormalizedLandmarks( + const mediapipe::NormalizedLandmarkList& proto) { + NormalizedLandmarks landmarks; + landmarks.landmarks.reserve(proto.landmark_size()); + for (const auto& landmark : proto.landmark()) { + landmarks.landmarks.push_back(ConvertToNormalizedLandmark(landmark)); + } + return landmarks; +} + +} // namespace mediapipe::tasks::components::containers diff --git a/mediapipe/tasks/cc/components/containers/landmark.h b/mediapipe/tasks/cc/components/containers/landmark.h new file mode 100644 index 00000000..15b73000 --- /dev/null +++ b/mediapipe/tasks/cc/components/containers/landmark.h @@ -0,0 +1,103 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_LANDMARK_H_ +#define MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_LANDMARK_H_ + +#include +#include +#include + +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe::tasks::components::containers { +constexpr float kLandmarkTolerance = 1e-6; + +// Landmark represents a point in 3D space with x, y, z coordinates. The +// landmark coordinates are in meters. z represents the landmark depth, and the +// smaller the value the closer the world landmark is to the camera. +struct Landmark { + float x; + float y; + float z; + // Landmark visibility. Should stay unset if not supported. + // Float score of whether landmark is visible or occluded by other objects. + // Landmark considered as invisible also if it is not present on the screen + // (out of scene bounds). Depending on the model, visibility value is either a + // sigmoid or an argument of sigmoid. + std::optional visibility = std::nullopt; + // Landmark presence. Should stay unset if not supported. + // Float score of whether landmark is present on the scene (located within + // scene bounds). Depending on the model, presence value is either a result of + // sigmoid or an argument of sigmoid function to get landmark presence + // probability. + std::optional presence = std::nullopt; + // Landmark name. Should stay unset if not supported. + std::optional name = std::nullopt; +}; + +inline bool operator==(const Landmark& lhs, const Landmark& rhs) { + return abs(lhs.x - rhs.x) < kLandmarkTolerance && + abs(lhs.y - rhs.y) < kLandmarkTolerance && + abs(lhs.z - rhs.z) < kLandmarkTolerance; +} + +// A normalized version of above Landmark struct. All coordinates should be +// within [0, 1]. +struct NormalizedLandmark { + float x; + float y; + float z; + std::optional visibility = std::nullopt; + std::optional presence = std::nullopt; + std::optional name = std::nullopt; +}; + +inline bool operator==(const NormalizedLandmark& lhs, + const NormalizedLandmark& rhs) { + return abs(lhs.x - rhs.x) < kLandmarkTolerance && + abs(lhs.y - rhs.y) < kLandmarkTolerance && + abs(lhs.z - rhs.z) < kLandmarkTolerance; +} + +// A list of Landmarks. +struct Landmarks { + std::vector landmarks; +}; + +// A list of NormalizedLandmarks. +struct NormalizedLandmarks { + std::vector landmarks; +}; + +// Utility function to convert from Landmark proto to Landmark struct. +Landmark ConvertToLandmark(const mediapipe::Landmark& proto); + +// Utility function to convert from NormalizedLandmark proto to +// NormalizedLandmark struct. +NormalizedLandmark ConvertToNormalizedLandmark( + const mediapipe::NormalizedLandmark& proto); + +// Utility function to convert from LandmarkList proto to Landmarks struct. +Landmarks ConvertToLandmarks(const mediapipe::LandmarkList& proto); + +// Utility function to convert from NormalizedLandmarkList proto to +// NormalizedLandmarks struct. +NormalizedLandmarks ConvertToNormalizedLandmarks( + const mediapipe::NormalizedLandmarkList& proto); + +} // namespace mediapipe::tasks::components::containers + +#endif // MEDIAPIPE_TASKS_CC_COMPONENTS_CONTAINERS_LANDMARK_H_ diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD index 03ec45f7..2552e7a1 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD @@ -154,11 +154,14 @@ cc_library( cc_library( name = "hand_landmarker_result", + srcs = ["hand_landmarker_result.cc"], hdrs = ["hand_landmarker_result.h"], visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/cc/components/containers:classification_result", + "//mediapipe/tasks/cc/components/containers:landmark", ], ) diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc index 3bb1ee8d..ab66fe13 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc @@ -155,9 +155,13 @@ absl::StatusOr> HandLandmarker::Create( Packet hand_world_landmarks_packet = status_or_packets.value()[kHandWorldLandmarksStreamName]; result_callback( - {{handedness_packet.Get>(), - hand_landmarks_packet.Get>(), - hand_world_landmarks_packet.Get>()}}, + ConvertToHandLandmarkerResult( + /* handedness= */ handedness_packet + .Get>(), + /* hand_landmarks= */ + hand_landmarks_packet.Get>(), + /* hand_world_landmarks= */ + hand_world_landmarks_packet.Get>()), image_packet.Get(), hand_landmarks_packet.Timestamp().Value() / kMicroSecondsPerMilliSecond); @@ -193,15 +197,21 @@ absl::StatusOr HandLandmarker::Detect( if (output_packets[kHandLandmarksStreamName].IsEmpty()) { return {HandLandmarkerResult()}; } - return {{/* handedness= */ - {output_packets[kHandednessStreamName] - .Get>()}, - /* hand_landmarks= */ - {output_packets[kHandLandmarksStreamName] - .Get>()}, - /* hand_world_landmarks */ - {output_packets[kHandWorldLandmarksStreamName] - .Get>()}}}; + return ConvertToHandLandmarkerResult(/* handedness= */ + output_packets[kHandednessStreamName] + .Get>(), + /* hand_landmarks= */ + output_packets[kHandLandmarksStreamName] + .Get>(), + /* hand_world_landmarks */ + output_packets + [kHandWorldLandmarksStreamName] + .Get>()); } absl::StatusOr HandLandmarker::DetectForVideo( @@ -228,17 +238,21 @@ absl::StatusOr HandLandmarker::DetectForVideo( if (output_packets[kHandLandmarksStreamName].IsEmpty()) { return {HandLandmarkerResult()}; } - return { - {/* handedness= */ - {output_packets[kHandednessStreamName] - .Get>()}, - /* hand_landmarks= */ - {output_packets[kHandLandmarksStreamName] - .Get>()}, - /* hand_world_landmarks */ - {output_packets[kHandWorldLandmarksStreamName] - .Get>()}}, - }; + return ConvertToHandLandmarkerResult(/* handedness= */ + output_packets[kHandednessStreamName] + .Get>(), + /* hand_landmarks= */ + output_packets[kHandLandmarksStreamName] + .Get>(), + /* hand_world_landmarks */ + output_packets + [kHandWorldLandmarksStreamName] + .Get>()); } absl::Status HandLandmarker::DetectAsync( diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.cc new file mode 100644 index 00000000..9d2ae2be --- /dev/null +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.cc @@ -0,0 +1,56 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h" + +#include + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace hand_landmarker { + +HandLandmarkerResult ConvertToHandLandmarkerResult( + const std::vector& handedness_proto, + const std::vector& hand_landmarks_proto, + const std::vector& hand_world_landmarks_proto) { + HandLandmarkerResult result; + result.handedness.resize(handedness_proto.size()); + result.hand_landmarks.resize(hand_landmarks_proto.size()); + result.hand_world_landmarks.resize(hand_world_landmarks_proto.size()); + std::transform(handedness_proto.begin(), handedness_proto.end(), + result.handedness.begin(), + [](const mediapipe::ClassificationList& classification_list) { + return components::containers::ConvertToClassifications( + classification_list); + }); + std::transform(hand_landmarks_proto.begin(), hand_landmarks_proto.end(), + result.hand_landmarks.begin(), + components::containers::ConvertToNormalizedLandmarks); + std::transform(hand_world_landmarks_proto.begin(), + hand_world_landmarks_proto.end(), + result.hand_world_landmarks.begin(), + components::containers::ConvertToLandmarks); + return result; +} + +} // namespace hand_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h index 5e51c244..1bca8e66 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h @@ -1,4 +1,4 @@ -/* Copyright 2022 The MediaPipe Authors. All Rights Reserved. +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ limitations under the License. #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" namespace mediapipe { namespace tasks { @@ -28,13 +30,18 @@ namespace hand_landmarker { // element represents a single hand detected in the image. struct HandLandmarkerResult { // Classification of handedness. - std::vector handedness; + std::vector handedness; // Detected hand landmarks in normalized image coordinates. - std::vector hand_landmarks; + std::vector hand_landmarks; // Detected hand landmarks in world coordinates. - std::vector hand_world_landmarks; + std::vector hand_world_landmarks; }; +HandLandmarkerResult ConvertToHandLandmarkerResult( + const std::vector& handedness_proto, + const std::vector& hand_landmarks_proto, + const std::vector& hand_world_landmarks_proto); + } // namespace hand_landmarker } // namespace vision } // namespace tasks diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result_test.cc new file mode 100644 index 00000000..109749b0 --- /dev/null +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result_test.cc @@ -0,0 +1,88 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_result.h" + +#include +#include + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace hand_landmarker { + +TEST(ConvertFromProto, Succeeds) { + mediapipe::ClassificationList classification_list_proto; + mediapipe::Classification& classification_proto = + *classification_list_proto.add_classification(); + classification_proto.set_index(1); + classification_proto.set_score(0.5); + classification_proto.set_label("Left"); + classification_proto.set_display_name("Left_Hand"); + + mediapipe::NormalizedLandmarkList normalized_landmark_list_proto; + mediapipe::NormalizedLandmark& normalized_landmark_proto = + *normalized_landmark_list_proto.add_landmark(); + normalized_landmark_proto.set_x(0.1); + normalized_landmark_proto.set_y(0.2); + normalized_landmark_proto.set_z(0.3); + + mediapipe::LandmarkList landmark_list_proto; + mediapipe::Landmark& landmark_proto = *landmark_list_proto.add_landmark(); + landmark_proto.set_x(3.1); + landmark_proto.set_y(5.2); + landmark_proto.set_z(4.3); + + std::vector classification_lists = { + classification_list_proto}; + std::vector normalized_landmarks_lists = { + normalized_landmark_list_proto}; + std::vector landmarks_lists = {landmark_list_proto}; + + HandLandmarkerResult hand_landmarker_result = ConvertToHandLandmarkerResult( + classification_lists, normalized_landmarks_lists, landmarks_lists); + + EXPECT_EQ(hand_landmarker_result.handedness.size(), 1); + EXPECT_EQ(hand_landmarker_result.handedness[0].categories.size(), 1); + EXPECT_THAT( + hand_landmarker_result.handedness[0].categories[0], + testing::FieldsAre(1, testing::FloatEq(0.5), "Left", "Left_Hand")); + + EXPECT_EQ(hand_landmarker_result.hand_landmarks.size(), 1); + EXPECT_EQ(hand_landmarker_result.hand_landmarks[0].landmarks.size(), 1); + EXPECT_THAT(hand_landmarker_result.hand_landmarks[0].landmarks[0], + testing::FieldsAre(testing::FloatEq(0.1), testing::FloatEq(0.2), + testing::FloatEq(0.3), std::nullopt, + std::nullopt, std::nullopt)); + + EXPECT_EQ(hand_landmarker_result.hand_world_landmarks.size(), 1); + EXPECT_EQ(hand_landmarker_result.hand_world_landmarks[0].landmarks.size(), 1); + EXPECT_THAT(hand_landmarker_result.hand_world_landmarks[0].landmarks[0], + testing::FieldsAre(testing::FloatEq(3.1), testing::FloatEq(5.2), + testing::FloatEq(4.3), std::nullopt, + std::nullopt, std::nullopt)); +} + +} // namespace hand_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc index 94d1b1c1..b21f1bee 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_test.cc @@ -32,6 +32,8 @@ limitations under the License. #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/tasks/cc/common.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" #include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" #include "mediapipe/tasks/cc/components/containers/rect.h" #include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" @@ -50,18 +52,16 @@ namespace { using ::file::Defaults; using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::components::containers::ConvertToClassifications; +using ::mediapipe::tasks::components::containers::ConvertToNormalizedLandmarks; using ::mediapipe::tasks::components::containers::RectF; using ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; using ::mediapipe::tasks::vision::core::ImageProcessingOptions; -using ::testing::EqualsProto; using ::testing::HasSubstr; using ::testing::Optional; -using ::testing::Pointwise; using ::testing::TestParamInfo; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::proto::Approximately; -using ::testing::proto::Partially; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kHandLandmarkerBundleAsset[] = "hand_landmarker.task"; @@ -74,7 +74,6 @@ constexpr char kPointingUpImage[] = "pointing_up.jpg"; constexpr char kPointingUpRotatedImage[] = "pointing_up_rotated.jpg"; constexpr char kNoHandsImage[] = "cats_and_dogs.jpg"; -constexpr float kLandmarksFractionDiff = 0.03; // percentage constexpr float kLandmarksAbsMargin = 0.03; constexpr float kHandednessMargin = 0.05; @@ -101,13 +100,47 @@ HandLandmarkerResult GetExpectedHandLandmarkerResult( const auto landmarks_detection_result = GetLandmarksDetectionResult(file_name); expected_results.hand_landmarks.push_back( - landmarks_detection_result.landmarks()); + ConvertToNormalizedLandmarks(landmarks_detection_result.landmarks())); expected_results.handedness.push_back( - landmarks_detection_result.classifications()); + ConvertToClassifications(landmarks_detection_result.classifications())); } return expected_results; } +MATCHER_P2(HandednessMatches, expected_handedness, tolerance, "") { + for (int i = 0; i < arg.size(); i++) { + for (int j = 0; j < arg[i].categories.size(); j++) { + if (arg[i].categories[j].index != + expected_handedness[i].categories[j].index) { + return false; + } + if (std::abs(arg[i].categories[j].score - + expected_handedness[i].categories[j].score) > tolerance) { + return false; + } + if (arg[i].categories[j].category_name != + expected_handedness[i].categories[j].category_name) { + return false; + } + } + } + return true; +} + +MATCHER_P2(LandmarksMatches, expected_landmarks, toleration, "") { + for (int i = 0; i < arg.size(); i++) { + for (int j = 0; j < arg[i].landmarks.size(); j++) { + if (std::abs(arg[i].landmarks[j].x - + expected_landmarks[i].landmarks[j].x) > toleration || + std::abs(arg[i].landmarks[j].y - + expected_landmarks[i].landmarks[j].y) > toleration) { + return false; + } + } + } + return true; +} + void ExpectHandLandmarkerResultsCorrect( const HandLandmarkerResult& actual_results, const HandLandmarkerResult& expected_results) { @@ -119,16 +152,15 @@ void ExpectHandLandmarkerResultsCorrect( ASSERT_EQ(actual_landmarks.size(), expected_landmarks.size()); ASSERT_EQ(actual_handedness.size(), expected_handedness.size()); + if (actual_landmarks.empty()) { + return; + } + ASSERT_GE(actual_landmarks.size(), 1); - EXPECT_THAT( - actual_handedness, - Pointwise(Approximately(Partially(EqualsProto()), kHandednessMargin), - expected_handedness)); + EXPECT_THAT(actual_handedness, + HandednessMatches(expected_handedness, kHandednessMargin)); EXPECT_THAT(actual_landmarks, - Pointwise(Approximately(Partially(EqualsProto()), - /*margin=*/kLandmarksAbsMargin, - /*fraction=*/kLandmarksFractionDiff), - expected_landmarks)); + LandmarksMatches(expected_landmarks, kLandmarksAbsMargin)); } } // namespace