From ace56b502a7bb2435f69454b92a051762b732314 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 2 Jun 2023 15:32:19 -0700 Subject: [PATCH] Add FaceLandmarker iOS API PiperOrigin-RevId: 537424705 --- mediapipe/tasks/ios/BUILD | 8 + .../ios/test/vision/face_landmarker/BUILD | 71 ++++ .../face_landmarker/MPPFaceLandmarkerTests.mm | 351 ++++++++++++++++++ .../tasks/ios/vision/face_landmarker/BUILD | 25 ++ .../sources/MPPFaceLandmarker.h | 116 ++++++ .../sources/MPPFaceLandmarker.m | 203 ++++++++++ .../sources/MPPFaceLandmarkerOptions.h | 8 + .../sources/MPPFaceLandmarkerOptions.m | 3 + 8 files changed, 785 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/face_landmarker/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm create mode 100644 mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h create mode 100644 mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index fae60e24..29b0dd65 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -53,6 +53,7 @@ CALCULATORS_AND_GRAPHS = [ "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] @@ -80,6 +81,9 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetector.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorOptions.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorResult.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarker.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerOptions.h", + "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerResult.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", @@ -164,6 +168,9 @@ apple_static_xcframework( ":MPPFaceDetector.h", ":MPPFaceDetectorOptions.h", ":MPPFaceDetectorResult.h", + ":MPPFaceLandmarker.h", + ":MPPFaceLandmarkerOptions.h", + ":MPPFaceLandmarkerResult.h", ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", @@ -173,6 +180,7 @@ apple_static_xcframework( ], deps = [ "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarker", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD new file mode 100644 index 00000000..93ecc00f --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/BUILD @@ -0,0 +1,71 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPFaceLandmarkerObjcTestLibrary", + testonly = 1, + srcs = ["MPPFaceLandmarkerTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarker", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarkerResult", + "//third_party/apple_frameworks:UIKit", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPFaceLandmarkerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPFaceLandmarkerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm new file mode 100644 index 00000000..f8773fb2 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm @@ -0,0 +1,351 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#import + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" + +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ClassificationListProto = ::mediapipe::ClassificationList; +using FaceGeometryProto = ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + +static ResourceFileInfo *const kPortraitImage = + @{@"name" : @"portrait", @"type" : @"jpg", @"orientation" : @(UIImageOrientationUp)}; +static ResourceFileInfo *const kPortraitRotatedImage = + @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationRight)}; +static ResourceFileInfo *const kCatImage = @{@"name" : @"cat", @"type" : @"jpg"}; +static ResourceFileInfo *const kPortraitExpectedLandmarksName = + @{@"name" : @"portrait_expected_face_landmarks", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kPortraitExpectedBlendshapesName = + @{@"name" : @"portrait_expected_blendshapes", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kPortraitExpectedGeometryName = + @{@"name" : @"portrait_expected_face_geometry", @"type" : kPbFileExtension}; +static NSString *const kFaceLandmarkerModelName = @"face_landmarker_v2"; +static NSString *const kFaceLandmarkerWithBlendshapesModelName = + @"face_landmarker_v2_with_blendshapes"; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +constexpr float kLandmarkErrorThreshold = 0.03f; +constexpr float kBlendshapesErrorThreshold = 0.1f; +constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +@interface MPPFaceLandmarkerTests : XCTestCase { +} +@end + +@implementation MPPFaceLandmarkerTests + +#pragma mark General Tests + +- (void)testCreateFaceLandmarkerWithMissingModelPathFails { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:@"" extension:@""]; + + NSError *error = nil; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:&error]; + XCTAssertNil(faceLandmarker); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +#pragma mark Image Mode Tests + +- (void)testDetectWithImageModeAndPotraitSucceeds { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:kFaceLandmarkerModelName + extension:@"task"]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:nil]; + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + usingFaceLandmarker:faceLandmarker + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:NULL]; +} + +- (void)testDetectWithImageModeAndPotraitAndFacialTransformationMatrixesSucceeds { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.outputFacialTransformationMatrixes = YES; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + MPPTransformMatrix *expectedTransformationMatrix = [MPPFaceLandmarkerTests + expectedTransformationMatrixFromFileInfo:kPortraitExpectedGeometryName]; + [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + usingFaceLandmarker:faceLandmarker + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:expectedTransformationMatrix]; +} + +- (void)testDetectWithImageModeAndNoFaceSucceeds { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:kFaceLandmarkerModelName + extension:@"task"]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath + error:nil]; + XCTAssertNotNil(faceLandmarker); + + NSError *error; + MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage + error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceLandmarkerResult); + XCTAssertEqualObjects(faceLandmarkerResult.faceLandmarks, [NSArray array]); + XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); + XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); +} + +#pragma mark Video Mode Tests + +- (void)testDetectWithVideoModeAndPotraitSucceeds { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeVideo; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + NSArray *expectedLandmarks = + [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + for (int i = 0; i < 3; i++) { + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertFaceLandmarkerResult:faceLandmarkerResult + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:NULL + expectedTransformationMatrix:NULL]; + } +} + +#pragma mark Running Mode Tests + +- (void)testDetectFailsWithCallingWrongAPIInImageMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + NSError *videoAPICallError; + XCTAssertFalse([faceLandmarker detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoAPICallError]); + + NSError *expectedVideoAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoAPICallError, expectedVideoAPICallError); +} + +- (void)testDetectFailsWithCallingWrongAPIInVideoMode { + MPPFaceLandmarkerOptions *options = + [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + options.runningMode = MPPRunningModeVideo; + + MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + NSError *imageAPICallError; + XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + + NSError *expectedImageAPICallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageAPICallError, expectedImageAPICallError); +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = + [[NSBundle bundleForClass:[MPPFaceLandmarkerTests class]] pathForResource:fileName + ofType:extension]; + return filePath; +} + ++ (NSArray *)expectedLandmarksFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + NormalizedLandmarkListProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + NSMutableArray *landmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)proto.landmark_size()]; + for (const auto &landmarkProto : proto.landmark()) { + [landmarks addObject:[MPPNormalizedLandmark normalizedLandmarkWithProto:landmarkProto]]; + } + return landmarks; +} + ++ (MPPClassifications *)expectedBlendshapesFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + ClassificationListProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + return [MPPClassifications classificationsWithClassificationListProto:proto + headIndex:0 + headName:[NSString string]]; +} + ++ (MPPTransformMatrix *)expectedTransformationMatrixFromFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; + FaceGeometryProto proto; + if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { + return nil; + } + return [[MPPTransformMatrix alloc] initWithData:proto.pose_transform_matrix().packed_data().data() + rows:proto.pose_transform_matrix().rows() + columns:proto.pose_transform_matrix().cols()]; +} + +- (void)assertFaceLandmarkerResult:(MPPFaceLandmarkerResult *)faceLandmarkerResult + containsExpectedLandmarks:(NSArray *)expectedLandmarks + expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes + expectedTransformationMatrix:(nullable MPPTransformMatrix *)expectedTransformationMatrix { + NSArray *landmarks = faceLandmarkerResult.faceLandmarks[0]; + XCTAssertEqual(landmarks.count, expectedLandmarks.count); + for (int i = 0; i < landmarks.count; ++i) { + XCTAssertEqualWithAccuracy(landmarks[i].x, expectedLandmarks[i].x, kLandmarkErrorThreshold, + @"index i = %d", i); + XCTAssertEqualWithAccuracy(landmarks[i].y, expectedLandmarks[i].y, kLandmarkErrorThreshold, + @"index i = %d", i); + } + + if (expectedBlendshapes == NULL) { + XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); + } else { + MPPClassifications *blendshapes = faceLandmarkerResult.faceBlendshapes[0]; + NSArray *actualCategories = blendshapes.categories; + NSArray *expectedCategories = expectedBlendshapes.categories; + XCTAssertEqual(actualCategories.count, expectedCategories.count); + for (int i = 0; i < actualCategories.count; ++i) { + XCTAssertEqual(actualCategories[i].index, expectedCategories[i].index, @"index i = %d", i); + XCTAssertEqualWithAccuracy(actualCategories[i].score, expectedCategories[i].score, + kBlendshapesErrorThreshold, @"index i = %d", i); + XCTAssertEqualObjects(actualCategories[i].categoryName, expectedCategories[i].categoryName, + @"index i = %d", i); + XCTAssertEqualObjects(actualCategories[i].displayName, expectedCategories[i].displayName, + @"index i = %d", i); + } + } + + if (expectedTransformationMatrix == NULL) { + XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); + } else { + MPPTransformMatrix *actualTransformationMatrix = + faceLandmarkerResult.facialTransformationMatrixes[0]; + XCTAssertEqual(actualTransformationMatrix.rows, expectedTransformationMatrix.rows); + XCTAssertEqual(actualTransformationMatrix.columns, expectedTransformationMatrix.columns); + for (int i = 0; i < actualTransformationMatrix.rows * actualTransformationMatrix.columns; ++i) { + XCTAssertEqualWithAccuracy(actualTransformationMatrix.data[i], + expectedTransformationMatrix.data[i], + kFacialTransformationMatrixErrorThreshold, @"index i = %d", i); + } + } +} + +#pragma mark Face Landmarker Initializers + +- (MPPFaceLandmarkerOptions *)faceLandmarkerOptionsWithModelName:(NSString *)modelName { + NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:modelName extension:@"task"]; + MPPFaceLandmarkerOptions *faceLandmarkerOptions = [[MPPFaceLandmarkerOptions alloc] init]; + faceLandmarkerOptions.baseOptions.modelAssetPath = modelPath; + return faceLandmarkerOptions; +} + +- (void)assertCreateFaceLandmarkerWithOptions:(MPPFaceLandmarkerOptions *)faceLandmarkerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPFaceLandmarker *faceLandmarker = + [[MPPFaceLandmarker alloc] initWithOptions:faceLandmarkerOptions error:&error]; + XCTAssertNil(faceLandmarker); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Assert Detection Results + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { + UIImageOrientation orientation = (UIImageOrientation)[fileInfo[@"orientation"] intValue]; + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPFaceLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + return image; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(ResourceFileInfo *)fileInfo + usingFaceLandmarker:(MPPFaceLandmarker *)faceLandmarker + containsExpectedLandmarks: + (NSArray *)expectedLandmarks + expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes + expectedTransformationMatrix: + (nullable MPPTransformMatrix *)expectedTransformationMatrix { + MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + + NSError *error; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage + error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceLandmarkerResult); + + [self assertFaceLandmarkerResult:faceLandmarkerResult + containsExpectedLandmarks:expectedLandmarks + expectedBlendshapes:expectedBlendshapes + expectedTransformationMatrix:expectedTransformationMatrix]; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/vision/face_landmarker/BUILD index c4b17269..14f82237 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/face_landmarker/BUILD @@ -37,7 +37,32 @@ objc_library( srcs = ["sources/MPPFaceLandmarkerOptions.m"], hdrs = ["sources/MPPFaceLandmarkerOptions.h"], deps = [ + ":MPPFaceLandmarkerResult", "//mediapipe/tasks/ios/core:MPPTaskOptions", "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPFaceLandmarker", + srcs = ["sources/MPPFaceLandmarker.m"], + hdrs = ["sources/MPPFaceLandmarker.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + ":MPPFaceLandmarkerOptions", + ":MPPFaceLandmarkerResult", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h new file mode 100644 index 00000000..3d336702 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h @@ -0,0 +1,116 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs face landmark detection on images. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + */ +NS_SWIFT_NAME(FaceLandmarker) +@interface MPPFaceLandmarker : NSObject + +/** + * Creates a new instance of `MPPFaceLandmarker` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPFaceLandmarker`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * face landmaker. + * + * @return A new instance of `MPPFaceLandmarker` with the given model path. `nil` if there is an + * error in initializing the face landmaker. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPFaceLandmarker` from the given `MPPFaceLandmarkerOptions`. + * + * @param options The options of type `MPPFaceLandmarkerOptions` to use for configuring the + * `MPPFaceLandmarker`. + * @param error An optional error parameter populated when there is an error in initializing the + * face landmaker. + * + * @return A new instance of `MPPFaceLandmarker` with the given options. `nil` if there is an error + * in initializing the face landmaker. + */ +- (nullable instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs face landmark detection on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with + * `MPPRunningModeImage`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face landmark detection is to be performed. + * @param error An optional error parameter populated when there is an error in performing face + * landmark detection on the input image. + * + * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + */ +- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); + +/** + * Performs face landmark detection on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with + * `MPPRunningModeVideo`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face landmark detection is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * landmark detection on the input image. + * + * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + */ +- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m new file mode 100644 index 00000000..01a8df33 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.m @@ -0,0 +1,203 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h" +#import + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h" + +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; + +// Constants for the underlying MP Tasks Graph. See +// https://github.com/google/mediapipe/tree/master/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc +static NSString *const kLandmarksOutStreamName = @"landmarks_out"; +static NSString *const kLandmarksOutTag = @"NORM_LANDMARKS"; +static NSString *const kBlendshapesOutStreamName = @"blendshapes_out"; +static NSString *const kBlendshapesOutTag = @"BLENDSHAPES"; +static NSString *const kFaceGeometryOutStreamName = @"face_geometry_out"; +static NSString *const kFaceGeometryOutTag = @"FACE_GEOMETRY"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph"; +static NSString *const kTaskName = @"faceLandmarker"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectStreamName.cppString, normalizedRectPacket \ + } \ + } + +@interface MPPFaceLandmarker () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; +} + +@end + +@implementation MPPFaceLandmarker + +- (instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + NSArray *inputStreams = @[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] + ]; + + NSMutableArray *outputStreams = [NSMutableArray + arrayWithObjects:[NSString + stringWithFormat:@"%@:%@", kLandmarksOutTag, kLandmarksOutStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName], nil]; + if (options.outputFaceBlendshapes) { + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kBlendshapesOutTag, + kBlendshapesOutStreamName]]; + } + if (options.outputFacialTransformationMatrixes) { + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kFaceGeometryOutTag, + kFaceGeometryOutStreamName]]; + } + + MPPTaskInfo *taskInfo = + [[MPPTaskInfo alloc] initWithTaskGraphName:kTaskGraphName + inputStreams:inputStreams + outputStreams:outputStreams + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:nullptr + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPFaceLandmarkerOptions *options = [[MPPFaceLandmarkerOptions alloc] init]; + options.baseOptions.modelAssetPath = modelPath; + return [self initWithOptions:options error:error]; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceLandmarkerResult + faceLandmarkerResultWithLandmarksPacket:outputPacketMap + .value()[kLandmarksOutStreamName.cppString] + blendshapesPacket:outputPacketMap + .value()[kBlendshapesOutStreamName.cppString] + transformationMatrixesPacket:outputPacketMap + .value()[kFaceGeometryOutStreamName.cppString]]; +} + +- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return nil; + } + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceLandmarkerResult + faceLandmarkerResultWithLandmarksPacket:outputPacketMap + .value()[kLandmarksOutStreamName.cppString] + blendshapesPacket:outputPacketMap + .value()[kBlendshapesOutStreamName.cppString] + transformationMatrixesPacket:outputPacketMap + .value()[kFaceGeometryOutStreamName.cppString]]; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h index 873858a5..19413ed0 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h @@ -16,6 +16,7 @@ #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" NS_ASSUME_NONNULL_BEGIN @@ -59,6 +60,13 @@ NS_SWIFT_NAME(FaceLandmarkerOptions) */ @property(nonatomic) BOOL outputFaceBlendshapes; +/** + * Whether FaceLandmarker outputs facial transformation_matrix. Facial transformation matrix is used + * to transform the face landmarks in canonical face to the detected face, so that users can apply + * face effects on the detected landmarks. + */ +@property(nonatomic) BOOL outputFacialTransformationMatrixes; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m index 80350d12..cbc89908 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m @@ -24,6 +24,7 @@ _minFacePresenceConfidence = 0.5f; _minTrackingConfidence = 0.5f; _outputFaceBlendshapes = NO; + _outputFacialTransformationMatrixes = NO; } return self; } @@ -36,6 +37,8 @@ faceLandmarkerOptions.minFacePresenceConfidence = self.minFacePresenceConfidence; faceLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence; faceLandmarkerOptions.outputFaceBlendshapes = self.outputFaceBlendshapes; + faceLandmarkerOptions.outputFacialTransformationMatrixes = + self.outputFacialTransformationMatrixes; return faceLandmarkerOptions; }