From 0fd60285b5a0939f3d4ec81e013b262a554941c6 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 10 Apr 2023 19:23:32 +0530 Subject: [PATCH 1/8] Updated roi not allowed check in ios vision task runner --- mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 98d46f17..964cd38c 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -90,7 +90,7 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; (UIImageOrientation)imageOrientation ROIAllowed:(BOOL)ROIAllowed error:(NSError **)error { - if (CGRectEqualToRect(roi, CGRectZero) && !ROIAllowed) { + if (!CGRectEqualToRect(roi, CGRectZero) && !ROIAllowed) { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"This task doesn't support region-of-interest."]; From adfe47d456e106a0e22eb85e6dbd9b30b7114285 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 10 Apr 2023 19:24:58 +0530 Subject: [PATCH 2/8] Removed roi apis from iOS object detector --- .../sources/MPPObjectDetector.h | 79 ------------------- .../sources/MPPObjectDetector.mm | 56 +++++++------ 2 files changed, 30 insertions(+), 105 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index 58344d0c..8b01c667 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -109,28 +109,6 @@ NS_SWIFT_NAME(ObjectDetector) error:(NSError **)error NS_SWIFT_NAME(detect(image:)); -/** - * Performs object detectionon the provided `MPPImage` cropped to the specified region of - * interest. Rotation will be applied on the cropped image according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeImage`. - * - * @param image The `MPPImage` on which object detection is to be performed. - * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which - * object detection should be performed. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input image. - * - * @return An `MPPObjectDetectionResult` object that contains a list of detections, each detection - * has a bounding box that is expressed in the unrotated input frame of reference coordinates - * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying - * image data. - */ -- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(detect(image:regionOfInterest:)); - /** * Performs object detection on the provided video frame of type `MPPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of @@ -153,36 +131,6 @@ NS_SWIFT_NAME(ObjectDetector) error:(NSError **)error NS_SWIFT_NAME(detect(videoFrame:timestampMs:)); -/** - * Performs object detection on the provided video frame of type `MPPImage` cropped to the - * specified region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeVideo`. - * - * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must - * be monotonically increasing. - * - * @param image A live stream image data of type `MPPImage` on which object detection is to be - * performed. - * @param timestampMs The video frame's timestamp (in milliseconds). The input timestamps must be - * monotonically increasing. - * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which - * object detection should be performed. - * - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input image. - * - * @return An `MPPObjectDetectionResult` object that contains a list of detections, each detection - * has a bounding box that is expressed in the unrotated input frame of reference coordinates - * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying - * image data. - */ -- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image - timestampMs:(NSInteger)timestampMs - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(detect(videoFrame:timestampMs:regionOfInterest:)); - /** * Sends live stream image data of type `MPPImage` to perform object detection using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of @@ -206,33 +154,6 @@ NS_SWIFT_NAME(ObjectDetector) timestampMs:(NSInteger)timestampMs error:(NSError **)error NS_SWIFT_NAME(detectAsync(image:timestampMs:)); -/** - * Sends live stream image data of type `MPPImage` to perform object detection, cropped to the - * specified region of interest. Rotation will be applied according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeLiveStream`. Results are provided asynchronously via the `completion` callback - * provided in the `MPPObjectDetectorOptions`. - * - * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent - * to the object detector. The input timestamps must be monotonically increasing. - * - * @param image A live stream image data of type `MPPImage` on which object detection is to be - * performed. - * @param timestampMs The timestamp (in milliseconds) which indicates when the input image is sent - * to the object detector. The input timestamps must be monotonically increasing. - * @param roi A `CGRect` specifying the region of interest within the given live stream image data - * of type `MPPImage`, on which iobject detection should be performed. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input live stream image data. - * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. - */ -- (BOOL)detectAsyncInImage:(MPPImage *)image - timestampMs:(NSInteger)timestampMs - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(detectAsync(image:timestampMs:regionOfInterest:)); - - (instancetype)init NS_UNAVAILABLE; + (instancetype)new NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 53dcad4a..cca33981 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -158,12 +158,11 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG - (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image timestampMs:(NSInteger)timestampMs - regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero imageOrientation:image.orientation - ROIAllowed:YES + ROIAllowed:NO error:error]; if (!rect.has_value()) { return std::nullopt; @@ -184,16 +183,41 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG } - (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - return [self detectInImage:image regionOfInterest:CGRectZero error:error]; + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageOrientation:image.orientation + ROIAllowed:YES + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPObjectDetectionResult + objectDetectionResultWithDetectionsPacket:outputPacketMap + .value()[kDetectionsStreamName.cppString]]; } - (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image timestampMs:(NSInteger)timestampMs - regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image timestampMs:timestampMs - regionOfInterest:roi error:error]; if (!inputPacketMap.has_value()) { return nil; @@ -211,22 +235,11 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG .value()[kDetectionsStreamName.cppString]]; } -- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image - timestampMs:(NSInteger)timestampMs - error:(NSError **)error { - return [self detectInVideoFrame:image - timestampMs:timestampMs - regionOfInterest:CGRectZero - error:error]; -} - - (BOOL)detectAsyncInImage:(MPPImage *)image timestampMs:(NSInteger)timestampMs - regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image timestampMs:timestampMs - regionOfInterest:roi error:error]; if (!inputPacketMap.has_value()) { return NO; @@ -235,13 +248,4 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; } -- (BOOL)detectAsyncInImage:(MPPImage *)image - timestampMs:(NSInteger)timestampMs - error:(NSError **)error { - return [self detectAsyncInImage:image - timestampMs:timestampMs - regionOfInterest:CGRectZero - error:error]; -} - @end From a2bab54640902a4178f8d63c7f05c17c24267366 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 10 Apr 2023 19:25:37 +0530 Subject: [PATCH 3/8] Added iOS Object Detector Objective D tests --- .../ios/test/vision/object_detector/BUILD | 55 ++ .../object_detector/MPPObjectDetectorTests.m | 703 ++++++++++++++++++ 2 files changed, 758 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/object_detector/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m diff --git a/mediapipe/tasks/ios/test/vision/object_detector/BUILD b/mediapipe/tasks/ios/test/vision/object_detector/BUILD new file mode 100644 index 00000000..36e1afb2 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/object_detector/BUILD @@ -0,0 +1,55 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/tasks:ios/ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPObjectDetectorObjcTestLibrary", + testonly = 1, + srcs = ["MPPObjectDetectorTests.m"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", + ], +) + +ios_unit_test( + name = "MPPObjectDetectorObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPObjectDetectorObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m new file mode 100644 index 00000000..cb76de88 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -0,0 +1,703 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h" + +static NSString *const kModelName = @"coco_ssd_mobilenet_v1_1.0_quant_2018_06_29"; +static NSDictionary *const kCatsAndDogsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; +static NSDictionary *const kCatsAndDogsRotatedImage = + @{@"name" : @"cats_and_dogs_rotated", @"type" : @"jpg"}; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define PixelDifferenceTolerance 5.0f +#define ScoreDifferenceTolerance 1e-2f + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +#define AssertEqualCategoryArrays(categories, expectedCategories, detectionIndex) \ + XCTAssertEqual(categories.count, expectedCategories.count); \ + for (int j = 0; j < categories.count; j++) { \ + XCTAssertEqual(categories[j].index, expectedCategories[j].index, \ + @"detection Index = %d category array index j = %d", detectionIndex, j); \ + XCTAssertEqualWithAccuracy( \ + categories[j].score, expectedCategories[j].score, ScoreDifferenceTolerance, \ + @"detection Index = %d, category array index j = %d", detectionIndex, j); \ + XCTAssertEqualObjects(categories[j].categoryName, expectedCategories[j].categoryName, \ + @"detection Index = %d, category array index j = %d", detectionIndex, \ + j); \ + XCTAssertEqualObjects(categories[j].displayName, expectedCategories[j].displayName, \ + @"detection Index = %d, category array index j = %d", detectionIndex, \ + j); \ + \ + \ + } + +#define AssertApproximatelyEqualBoundingBoxes(boundingBox, expectedBoundingBox, idx) \ + XCTAssertEqualWithAccuracy(boundingBox.origin.x, expectedBoundingBox.origin.x, \ + PixelDifferenceTolerance, @"index i = %d", idx); \ + XCTAssertEqualWithAccuracy(boundingBox.origin.y, expectedBoundingBox.origin.y, \ + PixelDifferenceTolerance, @"index i = %d", idx); \ + XCTAssertEqualWithAccuracy(boundingBox.size.width, expectedBoundingBox.size.width, \ + PixelDifferenceTolerance, @"index i = %d", idx); \ + XCTAssertEqualWithAccuracy(boundingBox.size.height, expectedBoundingBox.size.height, \ + PixelDifferenceTolerance, @"index i = %d", idx); + +#define AssertEqualDetections(detection, expectedDetection, idx) \ + XCTAssertNotNil(detection); \ + AssertEqualCategoryArrays(detection.categories, expectedDetection.categories, idx); \ + AssertApproximatelyEqualBoundingBoxes(detection.boundingBox, expectedDetection.boundingBox, idx); + +#define AssertEqualDetectionArrays(detections, expectedDetections) \ + XCTAssertEqual(detections.count, expectedDetections.count); \ + for (int i = 0; i < detections.count; i++) { \ + AssertEqualDetections(detections[i], expectedDetections[i], i); \ + } + +#define AssertEqualObjectDetectionResults(objectDetectionResult, expectedObjectDetectionResult, \ + expectedDetectionCount) \ + XCTAssertNotNil(objectDetectionResult); \ + NSArray *detectionsSubsetToCompare; \ + XCTAssertEqual(objectDetectionResult.detections.count, expectedDetectionCount); \ + if (objectDetectionResult.detections.count > expectedObjectDetectionResult.detections.count) { \ + detectionsSubsetToCompare = [objectDetectionResult.detections \ + subarrayWithRange:NSMakeRange(0, expectedObjectDetectionResult.detections.count)]; \ + } else { \ + detectionsSubsetToCompare = objectDetectionResult.detections; \ + } \ + \ + AssertEqualDetectionArrays(detectionsSubsetToCompare, expectedObjectDetectionResult.detections); \ + XCTAssertEqual(objectDetectionResult.timestampMs, expectedObjectDetectionResult.timestampMs); + +@interface MPPObjectDetectorTests : XCTestCase +@end + +@implementation MPPObjectDetectorTests + +#pragma mark Results + ++ (MPPObjectDetectionResult *)expectedDetectionResultForCatsAndDogsImageWithTimestampMs: + (NSInteger)timestampMs { + NSArray *detections = @[ + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(608, 161, 381, 439) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.656250f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(57, 398, 392, 196) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.51171875f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(257, 395, 173, 202) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.48828125f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(363, 195, 330, 412) + keypoints:nil], + ]; + + return [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:timestampMs]; +} + +#pragma mark File + +- (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Object Detector Initializers + +- (MPPObjectDetectorOptions *)objectDetectorOptionsWithModelName:(NSString *)modelName { + NSString *modelPath = [self filePathWithName:modelName extension:@"tflite"]; + MPPObjectDetectorOptions *objectDetectorOptions = [[MPPObjectDetectorOptions alloc] init]; + objectDetectorOptions.baseOptions.modelAssetPath = modelPath; + + return objectDetectorOptions; +} + +- (void)assertCreateObjectDetectorWithOptions:(MPPObjectDetectorOptions *)objectDetectorOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPObjectDetector *objectDetector = + [[MPPObjectDetector alloc] initWithOptions:objectDetectorOptions error:&error]; + + XCTAssertNil(objectDetector); + AssertEqualErrors(error, expectedError); +} + +- (MPPObjectDetector *)objectDetectorWithOptionsSucceeds: + (MPPObjectDetectorOptions *)objectDetectorOptions { + MPPObjectDetector *objectDetector = + [[MPPObjectDetector alloc] initWithOptions:objectDetectorOptions error:nil]; + XCTAssertNotNil(objectDetector); + + return objectDetector; +} + +#pragma mark Assert Detection Results + +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPObjectDetectorTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"]]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo + orientation:(UIImageOrientation)orientation { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPObjectDetectorTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + + return image; +} + +- (void)assertResultsOfDetectInImage:(MPPImage *)mppImage + usingObjectDetector:(MPPObjectDetector *)objectDetector + maxResults:(NSInteger)maxResults + equalsObjectDetectionResult:(MPPObjectDetectionResult *)expectedObjectDetectionResult { + MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInImage:mppImage + error:nil]; + AssertEqualObjectDetectionResults( + objectDetectionResult, expectedObjectDetectionResult, + maxResults > 0 ? maxResults : objectDetectionResult.detections.count); +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(NSDictionary *)fileInfo + usingObjectDetector:(MPPObjectDetector *)objectDetector + maxResults:(NSInteger)maxResults + + equalsObjectDetectionResult: + (MPPObjectDetectionResult *)expectedObjectDetectionResult { + MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + + [self assertResultsOfDetectInImage:mppImage + usingObjectDetector:objectDetector + maxResults:maxResults + equalsObjectDetectionResult:expectedObjectDetectionResult]; +} + +#pragma mark General Tests + +- (void)testCreateObjectDetectorWithMissingModelPathFails { + NSString *modelPath = [self filePathWithName:@"" extension:@""]; + + NSError *error = nil; + MPPObjectDetector *objectDetector = [[MPPObjectDetector alloc] initWithModelPath:modelPath + error:&error]; + XCTAssertNil(objectDetector); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +- (void)testCreateObjectDetectorAllowlistAndDenylistFails { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.categoryAllowlist = @[ @"cat" ]; + options.categoryDenylist = @[ @"dog" ]; + + [self assertCreateObjectDetectorWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: `category_allowlist` and " + @"`category_denylist` are mutually exclusive options." + }]]; +} + +- (void)testDetectWithModelPathSucceeds { + NSString *modelPath = [self filePathWithName:kModelName extension:@"tflite"]; + MPPObjectDetector *objectDetector = [[MPPObjectDetector alloc] initWithModelPath:modelPath + error:nil]; + XCTAssertNotNil(objectDetector); + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:-1 + equalsObjectDetectionResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:0]]; +} + +- (void)testDetectWithOptionsSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:-1 + equalsObjectDetectionResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:0]]; +} + +- (void)testDetectWithMaxResultsSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + const NSInteger maxResults = 4; + options.maxResults = maxResults; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:maxResults + equalsObjectDetectionResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:0]]; +} + +- (void)testDetectWithScoreThresholdSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.scoreThreshold = 0.68f; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + NSArray *detections = @[ + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(608, 161, 381, 439) + keypoints:nil], + ]; + MPPObjectDetectionResult *expectedObjectDetectionResult = + [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:0]; + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:-1 + equalsObjectDetectionResult:expectedObjectDetectionResult]; +} + +- (void)testDetectWithCategoryAllowlistSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.categoryAllowlist = @[ @"cat" ]; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + NSArray *detections = @[ + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(608, 161, 381, 439) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.656250f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(57, 398, 392, 196) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.51171875f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(257, 395, 173, 202) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.48828125f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(363, 195, 330, 412) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.355469f categoryName:@"cat" displayName:nil], + ] + boundingBox:CGRectMake(275, 216, 610, 386) + keypoints:nil], + ]; + + MPPObjectDetectionResult *expectedDetectionResult = + [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:0]; + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:-1 + equalsObjectDetectionResult:expectedDetectionResult]; +} + +- (void)testDetectWithCategoryDenylistSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.categoryDenylist = @[ @"cat" ]; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + NSArray *detections = @[ + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 + score:0.476562f + categoryName:@"teddy bear" + displayName:nil], + ] + boundingBox:CGRectMake(780, 407, 314, 190) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 + score:0.390625f + categoryName:@"teddy bear" + displayName:nil], + ] + boundingBox:CGRectMake(90, 225, 568, 366) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 + score:0.367188f + categoryName:@"teddy bear" + displayName:nil], + ] + boundingBox:CGRectMake(888, 434, 187, 167) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 score:0.332031f categoryName:@"bed" displayName:nil], + ] + boundingBox:CGRectMake(79, 364, 1097, 224) + keypoints:nil], + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 + score:0.289062f + categoryName:@"teddy bear" + displayName:nil], + ] + boundingBox:CGRectMake(605, 398, 445, 199) + keypoints:nil], + ]; + + MPPObjectDetectionResult *expectedDetectionResult = + [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:0]; + + [self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage + usingObjectDetector:objectDetector + maxResults:-1 + equalsObjectDetectionResult:expectedDetectionResult]; +} + +- (void)testDetectWithOrientationSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.maxResults = 1; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + NSArray *detections = @[ + [[MPPDetection alloc] initWithCategories:@[ + [[MPPCategory alloc] initWithIndex:-1 + score:0.750000f + categoryName:@"teddy bear" + displayName:nil], + ] + boundingBox:CGRectMake(0, 372, 416, 276) + keypoints:nil], + ]; + + MPPObjectDetectionResult *expectedDetectionResult = + [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:0]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsRotatedImage + orientation:UIImageOrientationRight]; + + [self assertResultsOfDetectInImage:image + usingObjectDetector:objectDetector + maxResults:1 + equalsObjectDetectionResult:expectedDetectionResult]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateObjectDetectorFailsWithResultListenerInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + options.runningMode = runningModesToTest[i]; + options.completion = + ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { + }; + + [self + assertCreateObjectDetectorWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode, a " + @"user-defined result callback should not be provided." + }]]; + } +} + +- (void)testCreateObjectDetectorFailsWithMissingResultListenerInLiveStreamMode { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateObjectDetectorWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode, a " + @"user-defined result callback must be provided." + }]]; +} + +- (void)testDetectFailsWithCallingWrongApiInImageMode { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([objectDetector detectAsyncInImage:image + timestampMs:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([objectDetector detectInVideoFrame:image timestampMs:0 error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInVideoMode { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + options.runningMode = MPPRunningModeVideo; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([objectDetector detectAsyncInImage:image + timestampMs:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + options.runningMode = MPPRunningModeLiveStream; + options.completion = ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { + + }; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + NSError *imageApiCallError; + XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([objectDetector detectInVideoFrame:image timestampMs:0 error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testClassifyWithVideoModeSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + options.runningMode = MPPRunningModeVideo; + + NSInteger maxResults = 4; + options.maxResults = maxResults; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + for (int i = 0; i < 3; i++) { + MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInVideoFrame:image + timestampMs:i + error:nil]; + AssertEqualObjectDetectionResults( + objectDetectionResult, + [MPPObjectDetectorTests expectedDetectionResultForCatsAndDogsImageWithTimestampMs:i], + maxResults); + } +} + +- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + NSInteger maxResults = 4; + options.maxResults = maxResults; + + options.runningMode = MPPRunningModeLiveStream; + + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = 1; + + options.completion = ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { + AssertEqualObjectDetectionResults( + result, + [MPPObjectDetectorTests expectedDetectionResultForCatsAndDogsImageWithTimestampMs:1], + maxResults); + [expectation fulfill]; + }; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + XCTAssertTrue([objectDetector detectAsyncInImage:image timestampMs:1 error:nil]); + + NSError *error; + XCTAssertFalse([objectDetector detectAsyncInImage:image timestampMs:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); + [self waitForExpectations:@[ expectation ] timeout:0.1]; +} + +- (void)testDetectWithLiveStreamModeSucceeds { + MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; + + NSInteger maxResults = 4; + options.maxResults = maxResults; + + options.runningMode = MPPRunningModeLiveStream; + + NSInteger iterationCount = 100; + + // Because of flow limiting, we cannot ensure that the callback will be + // invoked `iterationCount` times. + // An normal expectation will fail if expectation.fullfill() is not called + // `expectation.expectedFulfillmentCount` times. + // If `expectation.isInverted = true`, the test will only succeed if + // expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // Since in our case we cannot predict how many times the expectation is + // supposed to be fullfilled setting, + // `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if + // expectation is fullfilled <= `iterationCount` times. + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + options.completion = ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { + AssertEqualObjectDetectionResults( + result, + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:timestampMs], + maxResults); + [expectation fulfill]; + }; + + MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used + // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type + // `CMSampleBuffer`. + MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([objectDetector detectAsyncInImage:image timestampMs:i error:nil]); + } + + [self waitForExpectations:@[ expectation ] timeout:0.5]; +} + +@end From d06cf68c70189a04cc0f184b0b4164393bb8c4a4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 10 Apr 2023 19:28:46 +0530 Subject: [PATCH 4/8] Removed detect in image with region of interest api from iOS Object Detector --- .../sources/MPPObjectDetector.mm | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index cca33981..49935355 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -123,39 +123,6 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG return [self initWithOptions:options error:error]; } -- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image - regionOfInterest:(CGRect)roi - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi - imageOrientation:image.orientation - ROIAllowed:YES - error:error]; - if (!rect.has_value()) { - return nil; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPObjectDetectionResult - objectDetectionResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; -} - - (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image timestampMs:(NSInteger)timestampMs error:(NSError **)error { From 089361cd8953ce014d2300b8373c117064b3bdec Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 11 Apr 2023 17:37:54 +0530 Subject: [PATCH 5/8] Split macros into helpers in Objective C Tests --- .../object_detector/MPPObjectDetectorTests.m | 139 +++++++++--------- 1 file changed, 73 insertions(+), 66 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index cb76de88..67551c98 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -23,9 +23,8 @@ static NSDictionary *const kCatsAndDogsImage = @{@"name" : @"cats_and_dogs", @"t static NSDictionary *const kCatsAndDogsRotatedImage = @{@"name" : @"cats_and_dogs_rotated", @"type" : @"jpg"}; static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; - -#define PixelDifferenceTolerance 5.0f -#define ScoreDifferenceTolerance 1e-2f +static const float pixelDifferenceTolerance = 5.0f; +static const float scoreDifferenceTolerance = 1e-2f; #define AssertEqualErrors(error, expectedError) \ XCTAssertNotNil(error); \ @@ -35,59 +34,29 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ NSNotFound) -#define AssertEqualCategoryArrays(categories, expectedCategories, detectionIndex) \ - XCTAssertEqual(categories.count, expectedCategories.count); \ - for (int j = 0; j < categories.count; j++) { \ - XCTAssertEqual(categories[j].index, expectedCategories[j].index, \ - @"detection Index = %d category array index j = %d", detectionIndex, j); \ - XCTAssertEqualWithAccuracy( \ - categories[j].score, expectedCategories[j].score, ScoreDifferenceTolerance, \ - @"detection Index = %d, category array index j = %d", detectionIndex, j); \ - XCTAssertEqualObjects(categories[j].categoryName, expectedCategories[j].categoryName, \ - @"detection Index = %d, category array index j = %d", detectionIndex, \ - j); \ - XCTAssertEqualObjects(categories[j].displayName, expectedCategories[j].displayName, \ - @"detection Index = %d, category array index j = %d", detectionIndex, \ - j); \ - \ - \ - } +#define AssertEqualCategories(category, expectedCategory, detectionIndex, categoryIndex) \ + XCTAssertEqual(category.index, expectedCategory.index, \ + @"detection Index = %d category array index j = %d", detectionIndex, \ + categoryIndex); \ + XCTAssertEqualWithAccuracy(category.score, expectedCategory.score, scoreDifferenceTolerance, \ + @"detection Index = %d, category array index j = %d", detectionIndex, \ + categoryIndex); \ + XCTAssertEqualObjects(category.categoryName, expectedCategory.categoryName, \ + @"detection Index = %d, category array index j = %d", detectionIndex, \ + categoryIndex); \ + XCTAssertEqualObjects(category.displayName, expectedCategory.displayName, \ + @"detection Index = %d, category array index j = %d", detectionIndex, \ + categoryIndex); #define AssertApproximatelyEqualBoundingBoxes(boundingBox, expectedBoundingBox, idx) \ XCTAssertEqualWithAccuracy(boundingBox.origin.x, expectedBoundingBox.origin.x, \ - PixelDifferenceTolerance, @"index i = %d", idx); \ + pixelDifferenceTolerance, @"index i = %d", idx); \ XCTAssertEqualWithAccuracy(boundingBox.origin.y, expectedBoundingBox.origin.y, \ - PixelDifferenceTolerance, @"index i = %d", idx); \ + pixelDifferenceTolerance, @"index i = %d", idx); \ XCTAssertEqualWithAccuracy(boundingBox.size.width, expectedBoundingBox.size.width, \ - PixelDifferenceTolerance, @"index i = %d", idx); \ + pixelDifferenceTolerance, @"index i = %d", idx); \ XCTAssertEqualWithAccuracy(boundingBox.size.height, expectedBoundingBox.size.height, \ - PixelDifferenceTolerance, @"index i = %d", idx); - -#define AssertEqualDetections(detection, expectedDetection, idx) \ - XCTAssertNotNil(detection); \ - AssertEqualCategoryArrays(detection.categories, expectedDetection.categories, idx); \ - AssertApproximatelyEqualBoundingBoxes(detection.boundingBox, expectedDetection.boundingBox, idx); - -#define AssertEqualDetectionArrays(detections, expectedDetections) \ - XCTAssertEqual(detections.count, expectedDetections.count); \ - for (int i = 0; i < detections.count; i++) { \ - AssertEqualDetections(detections[i], expectedDetections[i], i); \ - } - -#define AssertEqualObjectDetectionResults(objectDetectionResult, expectedObjectDetectionResult, \ - expectedDetectionCount) \ - XCTAssertNotNil(objectDetectionResult); \ - NSArray *detectionsSubsetToCompare; \ - XCTAssertEqual(objectDetectionResult.detections.count, expectedDetectionCount); \ - if (objectDetectionResult.detections.count > expectedObjectDetectionResult.detections.count) { \ - detectionsSubsetToCompare = [objectDetectionResult.detections \ - subarrayWithRange:NSMakeRange(0, expectedObjectDetectionResult.detections.count)]; \ - } else { \ - detectionsSubsetToCompare = objectDetectionResult.detections; \ - } \ - \ - AssertEqualDetectionArrays(detectionsSubsetToCompare, expectedObjectDetectionResult.detections); \ - XCTAssertEqual(objectDetectionResult.timestampMs, expectedObjectDetectionResult.timestampMs); + pixelDifferenceTolerance, @"index i = %d", idx); @interface MPPObjectDetectorTests : XCTestCase @end @@ -124,6 +93,39 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; return [[MPPObjectDetectionResult alloc] initWithDetections:detections timestampMs:timestampMs]; } +- (void)assertDetections:(NSArray *)detections + isEqualToExpectedDetections:(NSArray *)expectedDetections { + for (int i = 0; i < detections.count; i++) { + MPPDetection *detection = detections[i]; + XCTAssertNotNil(detection); + for (int j = 0; j < detection.categories.count; j++) { + AssertEqualCategories(detection.categories[j], expectedDetections[i].categories[j], i, j); + } + AssertApproximatelyEqualBoundingBoxes(detection.boundingBox, expectedDetections[i].boundingBox, + i); + } +} + +- (void)assertObjectDetectionResult:(MPPObjectDetectionResult *)objectDetectionResult + isEqualToExpectedResult:(MPPObjectDetectionResult *)expectedObjectDetectionResult + expectedDetectionsCount:(NSInteger)expectedDetectionsCount { + XCTAssertNotNil(objectDetectionResult); + + NSArray *detectionsSubsetToCompare; + XCTAssertEqual(objectDetectionResult.detections.count, expectedDetectionsCount); + if (objectDetectionResult.detections.count > expectedObjectDetectionResult.detections.count) { + detectionsSubsetToCompare = [objectDetectionResult.detections + subarrayWithRange:NSMakeRange(0, expectedObjectDetectionResult.detections.count)]; + } else { + detectionsSubsetToCompare = objectDetectionResult.detections; + } + + [self assertDetections:detectionsSubsetToCompare + isEqualToExpectedDetections:expectedObjectDetectionResult.detections]; + + XCTAssertEqual(objectDetectionResult.timestampMs, expectedObjectDetectionResult.timestampMs); +} + #pragma mark File - (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { @@ -189,9 +191,11 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; equalsObjectDetectionResult:(MPPObjectDetectionResult *)expectedObjectDetectionResult { MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInImage:mppImage error:nil]; - AssertEqualObjectDetectionResults( - objectDetectionResult, expectedObjectDetectionResult, - maxResults > 0 ? maxResults : objectDetectionResult.detections.count); + + [self assertObjectDetectionResult:objectDetectionResult + isEqualToExpectedResult:expectedObjectDetectionResult + expectedDetectionsCount:maxResults > 0 ? maxResults + : objectDetectionResult.detections.count]; } - (void)assertResultsOfDetectInImageWithFileInfo:(NSDictionary *)fileInfo @@ -604,10 +608,12 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInVideoFrame:image timestampMs:i error:nil]; - AssertEqualObjectDetectionResults( - objectDetectionResult, - [MPPObjectDetectorTests expectedDetectionResultForCatsAndDogsImageWithTimestampMs:i], - maxResults); + + [self + assertObjectDetectionResult:objectDetectionResult + isEqualToExpectedResult:[MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:i] + expectedDetectionsCount:maxResults]; } } @@ -624,10 +630,11 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; expectation.expectedFulfillmentCount = 1; options.completion = ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { - AssertEqualObjectDetectionResults( - result, - [MPPObjectDetectorTests expectedDetectionResultForCatsAndDogsImageWithTimestampMs:1], - maxResults); + [self assertObjectDetectionResult:result + isEqualToExpectedResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:timestampMs] + expectedDetectionsCount:maxResults]; [expectation fulfill]; }; @@ -678,11 +685,11 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; expectation.inverted = YES; options.completion = ^(MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error) { - AssertEqualObjectDetectionResults( - result, - [MPPObjectDetectorTests - expectedDetectionResultForCatsAndDogsImageWithTimestampMs:timestampMs], - maxResults); + [self assertObjectDetectionResult:result + isEqualToExpectedResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampMs:timestampMs] + expectedDetectionsCount:maxResults]; [expectation fulfill]; }; From 27353310c35aa8aaa14052f36e09b38f0653b395 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 11 Apr 2023 18:11:33 +0530 Subject: [PATCH 6/8] Updated normalized rect calculation for some angles in MPPVisionTaskRunner --- mediapipe/tasks/ios/vision/core/sources/MPPImage.h | 7 ++----- mediapipe/tasks/ios/vision/core/sources/MPPImage.m | 9 +++------ .../ios/vision/core/sources/MPPVisionTaskRunner.h | 3 +++ .../ios/vision/core/sources/MPPVisionTaskRunner.mm | 14 ++++++++++++-- .../object_detector/sources/MPPObjectDetector.mm | 2 ++ 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h index deffc97e..db865b0e 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h @@ -33,11 +33,8 @@ static const MPPImageSourceType MPPImageSourceTypeSampleBuffer = 2; NS_SWIFT_NAME(MPImage) @interface MPPImage : NSObject -/** Width of the image in pixels. */ -@property(nonatomic, readonly) CGFloat width; - -/** Height of the image in pixels. */ -@property(nonatomic, readonly) CGFloat height; +/** Size of the image in pixels. */ +@property(nonatomic, readonly) CGSize size; /** * The display orientation of the image. If `imageSourceType` is `MPPImageSourceTypeImage`, the diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m index 1f5104ef..c8bba9ac 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m @@ -46,8 +46,7 @@ NS_ASSUME_NONNULL_BEGIN _imageSourceType = MPPImageSourceTypeImage; _orientation = orientation; _image = image; - _width = image.size.width * image.scale; - _height = image.size.height * image.scale; + _size = CGSizeMake(image.size.width * image.scale, image.size.height * image.scale); } return self; } @@ -72,8 +71,7 @@ NS_ASSUME_NONNULL_BEGIN _orientation = orientation; CVPixelBufferRetain(pixelBuffer); _pixelBuffer = pixelBuffer; - _width = CVPixelBufferGetWidth(pixelBuffer); - _height = CVPixelBufferGetHeight(pixelBuffer); + _size = CGSizeMake(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); } return self; } @@ -105,8 +103,7 @@ NS_ASSUME_NONNULL_BEGIN _orientation = orientation; CFRetain(sampleBuffer); _sampleBuffer = sampleBuffer; - _width = CVPixelBufferGetWidth(imageBuffer); - _height = CVPixelBufferGetHeight(imageBuffer); + _size = CGSizeMake(CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer)); } return self; } diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index a7216840..92b5563e 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -70,6 +70,8 @@ NS_ASSUME_NONNULL_BEGIN * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals * `CGRectZero` if `ROIAllowed` is NO. Otherwise, an error will be returned. + * @param imageSize A `CGSize` specifying the size of the image within which normalized rect is + * calculated. * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. * Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`, @@ -83,6 +85,7 @@ NS_ASSUME_NONNULL_BEGIN */ - (std::optional) normalizedRectFromRegionOfInterest:(CGRect)roi + imageSize:(CGSize)imageSize imageOrientation:(UIImageOrientation)imageOrientation ROIAllowed:(BOOL)ROIAllowed error:(NSError **)error; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 964cd38c..89347048 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -86,6 +86,7 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; } - (std::optional)normalizedRectFromRegionOfInterest:(CGRect)roi + imageSize:(CGSize)imageSize imageOrientation: (UIImageOrientation)imageOrientation ROIAllowed:(BOOL)ROIAllowed @@ -102,8 +103,6 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; NormalizedRect normalizedRect; normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi)); normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi)); - normalizedRect.set_width(CGRectGetWidth(calculatedRoi)); - normalizedRect.set_height(CGRectGetHeight(calculatedRoi)); int rotationDegrees = 0; switch (imageOrientation) { @@ -134,6 +133,17 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; normalizedRect.set_rotation(rotationDegrees * M_PI / kMPPOrientationDegreesDown); + if (rotationDegrees % 180 == 0) { + normalizedRect.set_width(CGRectGetWidth(calculatedRoi)); + normalizedRect.set_height(CGRectGetHeight(calculatedRoi)); + } else { + const float width = CGRectGetHeight(calculatedRoi) * imageSize.height / imageSize.width; + const float height = CGRectGetWidth(calculatedRoi) * imageSize.width / imageSize.height; + + normalizedRect.set_width(width); + normalizedRect.set_height(height); + } + return normalizedRect; } diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 49935355..16bcf5d7 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -128,6 +128,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG error:(NSError **)error { std::optional rect = [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:image.size imageOrientation:image.orientation ROIAllowed:NO error:error]; @@ -152,6 +153,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG - (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image error:(NSError **)error { std::optional rect = [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:image.size imageOrientation:image.orientation ROIAllowed:YES error:error]; From 114a11dc4e87d4fc69a47e83ef27262694a386b4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 11 Apr 2023 18:11:58 +0530 Subject: [PATCH 7/8] Updated iOS tests to reflect the new orientation calculation. --- .../test/vision/object_detector/MPPObjectDetectorTests.m | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index 67551c98..e68b48f0 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -421,12 +421,9 @@ static const float scoreDifferenceTolerance = 1e-2f; NSArray *detections = @[ [[MPPDetection alloc] initWithCategories:@[ - [[MPPCategory alloc] initWithIndex:-1 - score:0.750000f - categoryName:@"teddy bear" - displayName:nil], + [[MPPCategory alloc] initWithIndex:-1 score:0.699219f categoryName:@"cat" displayName:nil], ] - boundingBox:CGRectMake(0, 372, 416, 276) + boundingBox:CGRectMake(0, 608, 439, 387) keypoints:nil], ]; From 0fcf92d7d5c2fac1cfb549bbf223f0bafd4bb20a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 11 Apr 2023 18:18:35 +0530 Subject: [PATCH 8/8] Updated iOS Image Classifier to reflect new calculation for normalized rect --- .../image_classifier/MPPImageClassifierTests.m | 14 +++++++------- .../image_classifier/sources/MPPImageClassifier.mm | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m index f7b26837..36a62d99 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -387,16 +387,16 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; NSArray *expectedCategories = @[ [[MPPCategory alloc] initWithIndex:934 - score:0.622074f + score:0.753852f categoryName:@"cheeseburger" displayName:nil], - [[MPPCategory alloc] initWithIndex:963 - score:0.051214f - categoryName:@"meat loaf" - displayName:nil], [[MPPCategory alloc] initWithIndex:925 - score:0.048719f + score:0.028609f categoryName:@"guacamole" + displayName:nil], + [[MPPCategory alloc] initWithIndex:932 + score:0.027782f + categoryName:@"bagel" displayName:nil] ]; @@ -420,7 +420,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; NSArray *expectedCategories = @[ [[MPPCategory alloc] initWithIndex:560 - score:0.682305f + score:0.604605f categoryName:@"folding chair" displayName:nil] ]; diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 8051fbf3..b9bde841 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -120,6 +120,7 @@ static NSString *const kTaskGraphName = error:(NSError **)error { std::optional rect = [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageSize:image.size imageOrientation:image.orientation ROIAllowed:YES error:error]; @@ -154,6 +155,7 @@ static NSString *const kTaskGraphName = error:(NSError **)error { std::optional rect = [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageSize:image.size imageOrientation:image.orientation ROIAllowed:YES error:error];