diff --git a/mediapipe/tasks/cc/vision/face_landmarker/BUILD b/mediapipe/tasks/cc/vision/face_landmarker/BUILD index 8dc4e039..7b0994db 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/face_landmarker/BUILD @@ -114,3 +114,31 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "face_landmarker_result", + srcs = ["face_landmarker_result.cc"], + hdrs = ["face_landmarker_result.h"], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/tasks/cc/components/containers:classification_result", + "//mediapipe/tasks/cc/components/containers:landmark", + ], +) + +cc_library( + name = "face_landmarker_result_cc", + srcs = ["face_landmarker_result.cc"], + hdrs = ["face_landmarker_result.h"], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/tasks/cc/components/containers:classification_result", + "//mediapipe/tasks/cc/components/containers:landmark", + ], +) diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.cc new file mode 100644 index 00000000..3f369cc1 --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.cc @@ -0,0 +1,73 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" + +#include + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_landmarker { + +FaceLandmarkerResult ConvertToFaceLandmarkerResult( + std::vector face_landmarks_proto, + std::optional> + face_blendshapes_proto, + std::optional> + facial_transformation_matrix_proto) { + FaceLandmarkerResult result; + result.face_landmarks.resize(face_landmarks_proto.size()); + std::transform(face_landmarks_proto.begin(), face_landmarks_proto.end(), + result.face_landmarks.begin(), + components::containers::ConvertToNormalizedLandmarks); + if (face_blendshapes_proto.has_value()) { + result.face_blendshapes = + std::vector( + face_blendshapes_proto->size()); + std::transform( + face_blendshapes_proto->begin(), face_blendshapes_proto->end(), + result.face_blendshapes->begin(), + [](const mediapipe::ClassificationList& classification_list) { + return components::containers::ConvertToClassifications( + classification_list); + }); + } + if (facial_transformation_matrix_proto.has_value()) { + result.facial_transformation_matrix = + std::vector(facial_transformation_matrix_proto->size()); + std::transform(facial_transformation_matrix_proto->begin(), + facial_transformation_matrix_proto->end(), + result.facial_transformation_matrix->begin(), + [](const mediapipe::MatrixData& matrix_proto) { + mediapipe::Matrix matrix; + MatrixFromMatrixDataProto(matrix_proto, &matrix); + return matrix; + }); + } + return result; +} + +} // namespace face_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h new file mode 100644 index 00000000..9774d80d --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h @@ -0,0 +1,59 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKER_RESULT_H_ +#define MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKER_RESULT_H_ + +#include +#include + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_landmarker { + +// The face landmarks detection result from FaceLandmarker, where each vector +// element represents a single face detected in the image. +struct FaceLandmarkerResult { + // Detected hand landmarks in normalized image coordinates. + std::vector face_landmarks; + // Optional face blendshapes results. + std::optional> + face_blendshapes; + // Optional facial transformation matrix. + std::optional> facial_transformation_matrix; +}; + +// Convert face landmarks result from proto format to FaceLandmarkerResult. +FaceLandmarkerResult ConvertToFaceLandmarkerResult( + std::vector face_landmarks_proto, + std::optional> + face_blendshapes_proto = std::nullopt, + std::optional> + facial_transformation_matrix_proto = std::nullopt); + +} // namespace face_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKER_RESULT_H_ diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result_test.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result_test.cc new file mode 100644 index 00000000..c3ed2d37 --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result_test.cc @@ -0,0 +1,85 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" + +#include +#include + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_landmarker { + +class FaceLandmarkerResultTest : public ::testing::Test {}; + +TEST(FaceLandmarkerResultTest, Succeeds) { + mediapipe::NormalizedLandmarkList face_landmarks_proto; + mediapipe::NormalizedLandmark& normalized_landmark_proto = + *face_landmarks_proto.add_landmark(); + normalized_landmark_proto.set_x(0.1); + normalized_landmark_proto.set_y(0.2); + normalized_landmark_proto.set_z(0.3); + + mediapipe::ClassificationList face_blendshapes_proto; + mediapipe::Classification& classification_proto = + *face_blendshapes_proto.add_classification(); + classification_proto.set_index(1); + classification_proto.set_score(0.9075446); + classification_proto.set_label("browDownLeft"); + + mediapipe::MatrixData matrix_proto; + matrix_proto.set_rows(3); + matrix_proto.set_cols(3); + for (int i = 0; i < matrix_proto.rows() * matrix_proto.cols(); i++) { + matrix_proto.add_packed_data(i); + } + + FaceLandmarkerResult face_landmarker_result = ConvertToFaceLandmarkerResult( + {{face_landmarks_proto}}, {{face_blendshapes_proto}}, {{matrix_proto}}); + + EXPECT_EQ(face_landmarker_result.face_landmarks.size(), 1); + EXPECT_EQ(face_landmarker_result.face_landmarks[0].landmarks.size(), 1); + EXPECT_THAT(face_landmarker_result.face_landmarks[0].landmarks[0], + testing::FieldsAre(testing::FloatEq(0.1), testing::FloatEq(0.2), + testing::FloatEq(0.3), std::nullopt, + std::nullopt, std::nullopt)); + + ASSERT_TRUE(face_landmarker_result.face_blendshapes.has_value()); + EXPECT_EQ(face_landmarker_result.face_blendshapes->size(), 1); + EXPECT_EQ(face_landmarker_result.face_blendshapes->at(0).categories.size(), + 1); + EXPECT_THAT(face_landmarker_result.face_blendshapes->at(0).categories[0], + testing::FieldsAre(1, testing::FloatEq(0.9075446), "browDownLeft", + std::nullopt)); + + Matrix expected_matrix{{0, 3, 6}, {1, 4, 7}, {2, 5, 8}}; + ASSERT_TRUE(face_landmarker_result.facial_transformation_matrix.has_value()); + EXPECT_EQ(face_landmarker_result.facial_transformation_matrix->size(), 1); + EXPECT_EQ(face_landmarker_result.facial_transformation_matrix->at(0), + expected_matrix); +} + +} // namespace face_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe