From 2e4e17d83736e801167b534527ad06732bc19a1e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 13:13:03 +0530 Subject: [PATCH 01/18] Added MPPImage Utils for tests --- mediapipe/tasks/ios/test/vision/utils/BUILD | 13 +++++ .../vision/utils/sources/MPPImage+TestUtils.h | 51 +++++++++++++++++++ .../vision/utils/sources/MPPImage+TestUtils.m | 46 +++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/utils/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h create mode 100644 mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m diff --git a/mediapipe/tasks/ios/test/vision/utils/BUILD b/mediapipe/tasks/ios/test/vision/utils/BUILD new file mode 100644 index 00000000..cf7626db --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/utils/BUILD @@ -0,0 +1,13 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageTestUtils", + srcs = ["sources/MPPImage+TestUtils.m"], + hdrs = ["sources/MPPImage+TestUtils.h"], + module_name = "MPPImageTestUtils", + deps = [ + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h new file mode 100644 index 00000000..7f521e85 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h @@ -0,0 +1,51 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Helper utility for initializing `MPPImage` for MediaPipe iOS vision library tests. + */ +@interface MPPImage (TestUtils) +/** + * Loads an image from a file in an app bundle into a `MPPImage` object. + * + * @param classObject The specified class associated with the bundle containing + * the file to be loaded. + * @param name Name of the image file. + * @param type Extenstion of the image file. + * + * @return The `MPPImage` object contains the loaded image. This method returns + * nil if it cannot load the image. + */ ++ (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type + error:(NSError **)error + NS_SWIFT_NAME(imageFromBundle(class:filename:type:)); + ++ (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type + orientation:(UIImageOrientation)imageOrientation + error:(NSError **)error + NS_SWIFT_NAME(imageFromBundle(class:filename:type:orientation:)); + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m new file mode 100644 index 00000000..926d7a90 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m @@ -0,0 +1,46 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" + +@implementation MPPImage (TestUtils) + ++ (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type + error:(NSError **)error { + NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; + if (!imagePath) return nil; + + UIImage *image = [[UIImage alloc] initWithContentsOfFile:imagePath]; + if (!image) return nil; + + return [[MPPImage alloc] initWithUIImage:image error:error]; +} + ++ (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type + orientation:(UIImageOrientation)imageOrientation + error:(NSError **)error { + NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; + if (!imagePath) return nil; + + UIImage *image = [[UIImage alloc] initWithContentsOfFile:imagePath]; + if (!image) return nil; + + return [[MPPImage alloc] initWithUIImage:image orientation:imageOrientation error:error]; +} + +@end From 6e62c113fb5af39ec955d3cacfec08f50a41cfd7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:39:54 +0530 Subject: [PATCH 02/18] Fixed iOS running mode display strings --- mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h index ab76546d..ee52a693 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -44,9 +44,9 @@ NS_INLINE NSString *MPPRunningModeDisplayName(MPPRunningMode runningMode) { } NSString *displayNameMap[MPPRunningModeLiveStream + 1] = { - [MPPRunningModeImage] = @"#MPPRunningModeImage", - [MPPRunningModeVideo] = @ "#MPPRunningModeVideo", - [MPPRunningModeLiveStream] = @ "#MPPRunningModeLiveStream"}; + [MPPRunningModeImage] = @"Image", + [MPPRunningModeVideo] = @"Video", + [MPPRunningModeLiveStream] = @"Live Stream"}; return displayNameMap[runningMode]; } From 3227635ea0cece93bd09113815e4e4b16dc291b0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:40:15 +0530 Subject: [PATCH 03/18] Fixed bug in roi assignment --- .../tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 492d29a8..787dceea 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -97,7 +97,7 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; return std::nullopt; } - CGRect calculatedRoi = CGRectEqualToRect(roi, CGRectZero) ? roi : CGRectMake(0.0, 0.0, 1.0, 1.0); + CGRect calculatedRoi = CGRectEqualToRect(roi, CGRectZero) ? CGRectMake(0.0, 0.0, 1.0, 1.0) : roi; NormalizedRect normalizedRect; normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi)); @@ -138,7 +138,7 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; } - (std::optional)processImagePacketMap:(const PacketMap &)packetMap - error:(NSError **)error { + error:(NSError **)error { if (_runningMode != MPPRunningModeImage) { [MPPCommonUtils createCustomError:error From ddce0417257dd55e387fa34b9b10f5927e28abc1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:40:53 +0530 Subject: [PATCH 04/18] Fixed incorrect stride value in MPPImageUtils --- .../core/utils/sources/MPPImage+Utils.mm | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 87f9a8a3..0f55f645 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -133,6 +133,9 @@ using ::mediapipe::ImageFrame; size_t height = CVPixelBufferGetHeight(pixelBuffer); size_t stride = CVPixelBufferGetBytesPerRow(pixelBuffer); + size_t destinationChannelCount = 3; + size_t destinationStride = destinationChannelCount * width; + uint8_t *rgbPixelData = [MPPPixelDataUtils rgbPixelDataFromPixelData:(uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer) withWidth:CVPixelBufferGetWidth(pixelBuffer) @@ -148,7 +151,7 @@ using ::mediapipe::ImageFrame; } std::unique_ptr imageFrame = absl::make_unique( - ::mediapipe::ImageFormat::SRGB, width, height, stride, static_cast(rgbPixelData), + ::mediapipe::ImageFormat::SRGB, width, height, destinationStride, static_cast(rgbPixelData), /*deleter=*/free); return imageFrame; @@ -183,11 +186,14 @@ using ::mediapipe::ImageFrame; NSInteger bitsPerComponent = 8; NSInteger channelCount = 4; + size_t bytesPerRow = channelCount * width; + + NSInteger destinationChannelCount = 3; + size_t destinationBytesPerRow = destinationChannelCount * width; + UInt8 *pixelDataToReturn = NULL; CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); - size_t bytesPerRow = channelCount * width; - // iOS infers bytesPerRow if it is set to 0. // See https://developer.apple.com/documentation/coregraphics/1455939-cgbitmapcontextcreate // But for segmentation test image, this was not the case. @@ -219,9 +225,13 @@ using ::mediapipe::ImageFrame; CGColorSpaceRelease(colorSpace); + if (!pixelDataToReturn) { + return nullptr; + } + std::unique_ptr imageFrame = absl::make_unique(mediapipe::ImageFormat::SRGB, (int)width, (int)height, - (int)bytesPerRow, static_cast(pixelDataToReturn), + (int)destinationBytesPerRow, static_cast(pixelDataToReturn), /*deleter=*/free); return imageFrame; From 1c4be91a3aa6186fc2c88be81559b3da038b327b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:41:27 +0530 Subject: [PATCH 05/18] Fixed stream names in MPPImageClassifier --- .../image_classifier/sources/MPPImageClassifier.mm | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 0ad79003..c1c8d656 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -36,7 +36,7 @@ static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; static NSString *const kImageInStreamName = @"image_in"; static NSString *const kImageOutStreamName = @"image_out"; static NSString *const kImageTag = @"IMAGE"; -static NSString *const kNormRectName = @"norm_rect_in"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; static NSString *const kNormRectTag = @"NORM_RECT"; static NSString *const kTaskGraphName = @@ -44,7 +44,7 @@ static NSString *const kTaskGraphName = #define InputPacketMap(imagePacket, normalizedRectPacket) \ { \ - {kImageInStreamName.cppString, imagePacket}, { kNormRectName.cppString, normalizedRectPacket } \ + {kImageInStreamName.cppString, imagePacket}, { kNormRectStreamName.cppString, normalizedRectPacket } \ } @interface MPPImageClassifier () { @@ -61,9 +61,13 @@ static NSString *const kTaskGraphName = MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] initWithTaskGraphName:kTaskGraphName inputStreams:@[ [NSString - stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName] ] + stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString + stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] ] outputStreams:@[ [NSString stringWithFormat:@"%@:%@", kClassificationsTag, - kClassificationsStreamName] ] + kClassificationsStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, + kImageOutStreamName] ] taskOptions:options enableFlowLimiting:NO error:error]; From 960e7a62835ebddecba0ed5ab9be72f4adc2963a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:41:50 +0530 Subject: [PATCH 06/18] Fixed incorrect method call in MPPImageClassifier for Image Mode --- .../ios/vision/image_classifier/sources/MPPImageClassifier.mm | 1 - 1 file changed, 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index c1c8d656..ed71730d 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -134,7 +134,6 @@ static NSString *const kTaskGraphName = PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - std::optional outputPacketMap = [_visionTaskRunner processPacketMap:inputPacketMap error:error]; if (!outputPacketMap.has_value()) { return nil; From 190463228209700f0c3cb5d8ceac840dd0357e32 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:42:12 +0530 Subject: [PATCH 07/18] Fixed incorrect method call for image mode --- .../ios/vision/image_classifier/sources/MPPImageClassifier.mm | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index ed71730d..b817603f 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -134,6 +134,7 @@ static NSString *const kTaskGraphName = PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap error:error]; if (!outputPacketMap.has_value()) { return nil; From 8077743bfc7e40e5c95d536e0d7e2df1df93657c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:42:57 +0530 Subject: [PATCH 08/18] Linked in Opencv ios framework with vision tasks. --- mediapipe/tasks/ios/vision/core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index a8164d67..1efb3c33 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -54,6 +54,7 @@ objc_library( ], deps = [ ":MPPRunningMode", + "@ios_opencv//:OpencvFramework", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", From 48190e6600af6d9162c4a443e9810d613dc305ee Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:43:30 +0530 Subject: [PATCH 09/18] Updated method signatures in MPPImage+TestUtils --- .../vision/utils/sources/MPPImage+TestUtils.h | 15 ++++++- .../vision/utils/sources/MPPImage+TestUtils.m | 40 +++++++++++-------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h index 7f521e85..eb0edf6a 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h @@ -22,6 +22,7 @@ NS_ASSUME_NONNULL_BEGIN * Helper utility for initializing `MPPImage` for MediaPipe iOS vision library tests. */ @interface MPPImage (TestUtils) + /** * Loads an image from a file in an app bundle into a `MPPImage` object. * @@ -36,14 +37,24 @@ NS_ASSUME_NONNULL_BEGIN + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type - error:(NSError **)error NS_SWIFT_NAME(imageFromBundle(class:filename:type:)); +/** + * Loads an image from a file in an app bundle into a `MPPImage` object with the specified orientation. + * + * @param classObject The specified class associated with the bundle containing + * the file to be loaded. + * @param name Name of the image file. + * @param type Extenstion of the image file. + * @param orientation Orientation of the image. + * + * @return The `MPPImage` object contains the loaded image. This method returns + * nil if it cannot load the image. + */ + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type orientation:(UIImageOrientation)imageOrientation - error:(NSError **)error NS_SWIFT_NAME(imageFromBundle(class:filename:type:orientation:)); @end diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m index 926d7a90..5421fd3b 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m @@ -14,33 +14,41 @@ #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +@interface UIImage (FileUtils) + ++(nullable UIImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type; + +@end + +@implementation UIImage (FileUtils) + ++(nullable UIImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type { + + NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; + if (!imagePath) return nil; + + return [[UIImage alloc] initWithContentsOfFile:imagePath]; +} + +@end + @implementation MPPImage (TestUtils) + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name - ofType:(NSString *)type - error:(NSError **)error { - NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; - if (!imagePath) return nil; + ofType:(NSString *)type { + UIImage *image = [UIImage imageFromBundleWithClass:classObject fileName:name ofType:type]; - UIImage *image = [[UIImage alloc] initWithContentsOfFile:imagePath]; - if (!image) return nil; - - return [[MPPImage alloc] initWithUIImage:image error:error]; + return [[MPPImage alloc] initWithUIImage:image error:nil]; } + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type - orientation:(UIImageOrientation)imageOrientation - error:(NSError **)error { - NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; - if (!imagePath) return nil; + orientation:(UIImageOrientation)imageOrientation { + UIImage *image = [UIImage imageFromBundleWithClass:classObject fileName:name ofType:type]; - UIImage *image = [[UIImage alloc] initWithContentsOfFile:imagePath]; - if (!image) return nil; - - return [[MPPImage alloc] initWithUIImage:image orientation:imageOrientation error:error]; + return [[MPPImage alloc] initWithUIImage:image orientation:imageOrientation error:nil]; } @end From d4b60a781ef66d0591fbb31a6347348b755847a2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 18:44:01 +0530 Subject: [PATCH 10/18] Added MPPImageClassifier Objective C Tests --- .../ios/test/vision/image_classifier/BUILD | 66 ++ .../MPPImageClassifierTests.m | 682 ++++++++++++++++++ 2 files changed, 748 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/image_classifier/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/BUILD b/mediapipe/tasks/ios/test/vision/image_classifier/BUILD new file mode 100644 index 00000000..ef8e3955 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_classifier/BUILD @@ -0,0 +1,66 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_unit_test", +) +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_library", +) +load( + "//mediapipe/tasks:ios/ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageClassifierObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageClassifierTests.m"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + ], + sdk_frameworks = [ + "AVFoundation", + ] +) + +ios_unit_test( + name = "MPPImageClassifierObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageClassifierObjcTestLibrary", + ], + +) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m new file mode 100644 index 00000000..6be32575 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -0,0 +1,682 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h" + +static NSString *kFloatModelName = @"mobilenet_v2_1.0_224"; +static NSString *const kQuantizedModelName = @"mobilenet_v1_0.25_224_quant"; +static NSDictionary *const kBurgerImage = @{@"name" : @"burger", @"type" : @"jpg"}; +static NSDictionary *const kBurgerRotatedImage = @{@"name" : @"burger_rotated", @"type" : @"jpg"}; +static NSDictionary *const kMultiObjectsImage = @{@"name" : @"multi_objects", @"type" : @"jpg"}; +static NSDictionary *const kMultiObjectsRotatedImage = @{@"name" : @"multi_objects_rotated", @"type" : @"jpg"}; +static const int kMobileNetCategoriesCount = 1001; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +#define AssertEqualCategoryArrays(categories, expectedCategories) \ + XCTAssertEqual(categories.count, expectedCategories.count); \ + for (int i = 0; i < categories.count; i++) { \ + XCTAssertEqual(categories[i].index, expectedCategories[i].index, @"index i = %d", i); \ + XCTAssertEqualWithAccuracy(categories[i].score, expectedCategories[i].score, 1e-3, \ + @"index i = %d", i); \ + XCTAssertEqualObjects(categories[i].categoryName, expectedCategories[i].categoryName, \ + @"index i = %d", i); \ + XCTAssertEqualObjects(categories[i].displayName, expectedCategories[i].displayName, \ + @"index i = %d", i); \ + } + +#define AssertImageClassifierResultHasOneHead(imageClassifierResult) \ + XCTAssertNotNil(imageClassifierResult); \ + XCTAssertNotNil(imageClassifierResult.classificationResult); \ + XCTAssertEqual(imageClassifierResult.classificationResult.classifications.count, 1); \ + XCTAssertEqual(imageClassifierResult.classificationResult.classifications[0].headIndex, 0); + +@interface MPPImageClassifierTests : XCTestCase +@end + +@implementation MPPImageClassifierTests + +#pragma mark Results + ++ (NSArray *)expectedResultCategoriesForClassifyBurgerImageWithFloatModel { + return @[ + [[MPPCategory alloc] initWithIndex:934 + score:0.786005f + categoryName:@"cheeseburger" + displayName:nil], + [[MPPCategory alloc] initWithIndex:932 score:0.023508f categoryName:@"bagel" displayName:nil], + [[MPPCategory alloc] initWithIndex:925 + score:0.021172f + categoryName:@"guacamole" + displayName:nil] + ]; +} + +#pragma mark File + +- (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Classifier Initializers + +- (MPPImageClassifierOptions *)imageClassifierOptionsWithModelName:(NSString *)modelName { + NSString *modelPath = [self filePathWithName:modelName extension:@"tflite"]; + MPPImageClassifierOptions *imageClassifierOptions = [[MPPImageClassifierOptions alloc] init]; + imageClassifierOptions.baseOptions.modelAssetPath = modelPath; + + return imageClassifierOptions; +} + +- (MPPImageClassifier *)imageClassifierFromModelFileWithName:(NSString *)modelName { + NSString *modelPath = [self filePathWithName:modelName extension:@"tflite"]; + MPPImageClassifier *imageClassifier = [[MPPImageClassifier alloc] initWithModelPath:modelPath + error:nil]; + XCTAssertNotNil(imageClassifier); + + return imageClassifier; +} + +- (MPPImageClassifier *)imageClassifierWithOptionsSucceeds: + (MPPImageClassifierOptions *)imageClassifierOptions { + MPPImageClassifier *imageClassifier = + [[MPPImageClassifier alloc] initWithOptions:imageClassifierOptions error:nil]; + XCTAssertNotNil(imageClassifier); + + return imageClassifier; +} + +#pragma mark Assert Classify Results + +-(MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPImageClassifierTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"]]; + XCTAssertNotNil(image); + + return image; +} + +-(MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo orientation:(UIImageOrientation)orientation { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPImageClassifierTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + + return image; +} + +- (void)assertCreateImageClassifierWithOptions:(MPPImageClassifierOptions *)imageClassifierOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPImageClassifier *imageClassifier = + [[MPPImageClassifier alloc] initWithOptions:imageClassifierOptions error:&error]; + + XCTAssertNil(imageClassifier); + AssertEqualErrors(error, expectedError); +} + +- (void)assertImageClassifierResult:(MPPImageClassifierResult *)imageClassifierResult + hasExpectedCategoriesCount:(NSInteger)expectedCategoriesCount + expectedCategories:(NSArray *)expectedCategories { + AssertImageClassifierResultHasOneHead(imageClassifierResult); + + NSArray *resultCategories = + imageClassifierResult.classificationResult.classifications[0].categories; + XCTAssertEqual(resultCategories.count, expectedCategoriesCount); + + NSArray *categorySubsetToCompare; + if (resultCategories.count > expectedCategories.count) { + categorySubsetToCompare = + [resultCategories subarrayWithRange:NSMakeRange(0, expectedCategories.count)]; + } else { + categorySubsetToCompare = resultCategories; + } + AssertEqualCategoryArrays(categorySubsetToCompare, expectedCategories); +} + +- (void)assertResultsOfClassifyImage:(MPPImage *)mppImage + usingImageClassifier:(MPPImageClassifier *)imageClassifier + expectedCategoriesCount:(NSInteger)expectedCategoriesCount + equalsCategories:(NSArray *)expectedCategories { + MPPImageClassifierResult *imageClassifierResult = [imageClassifier classifyImage:mppImage + error:nil]; + + [self assertImageClassifierResult:imageClassifierResult + hasExpectedCategoriesCount:expectedCategoriesCount + expectedCategories:expectedCategories]; +} + +- (void)assertResultsOfClassifyImageWithFileInfo:(NSDictionary *)fileInfo + usingImageClassifier:(MPPImageClassifier *)imageClassifier + expectedCategoriesCount:(NSInteger)expectedCategoriesCount + equalsCategories:(NSArray *)expectedCategories { + MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + + [self assertResultsOfClassifyImage:mppImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategoriesCount + equalsCategories:expectedCategories]; +} + +#pragma mark General Tests + +- (void)testCreateImageClassifierWithMissingModelPathFails { + NSString *modelPath = [self filePathWithName:@"" extension:@""]; + + NSError *error = nil; + MPPImageClassifier *imageClassifier = [[MPPImageClassifier alloc] initWithModelPath:modelPath + error:&error]; + XCTAssertNil(imageClassifier); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +- (void)testCreateImageClassifierAllowlistAndDenylistFails { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + options.categoryAllowlist = @[ @"cheeseburger" ]; + options.categoryDenylist = @[ @"bagel" ]; + + [self assertCreateImageClassifierWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: `category_allowlist` and " + @"`category_denylist` are mutually exclusive options." + }]]; +} + +// - (void)testCreateImageClassifierFailsWithInvalidMaxResults { +// MPPImageClassifierOptions *options = +// [self imageClassifierOptionsWithModelName:kFloatModelName]; +// options.maxResults = 0; + +// [self assertCreateImageClassifierWithOptions:options +// failsWithExpectedError: +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"INVALID_ARGUMENT: Invalid `max_results` +// option: " +// @"value must be > 0." +// }]]; +// } + +- (void)testClassifyWithModelPathAndFloatModelSucceeds { + MPPImageClassifier *imageClassifier = [self imageClassifierFromModelFileWithName:kFloatModelName]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:kMobileNetCategoriesCount + equalsCategories:[MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; +} + +- (void)testClassifyWithOptionsAndFloatModelSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + const NSInteger maxResults = 3; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:maxResults + equalsCategories:[MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; +} + +- (void)testClassifyWithQuantizedModelSucceeds { + MPPImageClassifierOptions *options = + [self imageClassifierOptionsWithModelName:kQuantizedModelName]; + + const NSInteger maxResults = 1; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ [[MPPCategory alloc] initWithIndex:934 + score:0.972656f + categoryName:@"cheeseburger" + displayName:nil] ]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:maxResults + equalsCategories:expectedCategories]; +} + +- (void)testClassifyWithScoreThresholdSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.scoreThreshold = 0.25f; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ [[MPPCategory alloc] initWithIndex:934 + score:0.786005f + categoryName:@"cheeseburger" + displayName:nil] ]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; +} + +- (void)testClassifyWithAllowListSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.categoryAllowlist = @[ @"cheeseburger", @"guacamole", @"meat loaf" ]; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ + [[MPPCategory alloc] initWithIndex:934 + score:0.786005f + categoryName:@"cheeseburger" + displayName:nil], + [[MPPCategory alloc] initWithIndex:925 + score:0.021172f + categoryName:@"guacamole" + displayName:nil], + [[MPPCategory alloc] initWithIndex:963 + score:0.006279315f + categoryName:@"meat loaf" + displayName:nil], + + ]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; +} + +- (void)testClassifyWithDenyListSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.categoryDenylist = @[ + @"bagel", + ]; + options.maxResults = 3; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ + [[MPPCategory alloc] initWithIndex:934 + score:0.786005f + categoryName:@"cheeseburger" + displayName:nil], + [[MPPCategory alloc] initWithIndex:925 + score:0.021172f + categoryName:@"guacamole" + displayName:nil], + [[MPPCategory alloc] initWithIndex:963 + score:0.006279315f + categoryName:@"meat loaf" + displayName:nil], + + ]; + + [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; +} + +- (void)testClassifyWithRegionOfInterestSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + NSInteger maxResults = 1; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ [[MPPCategory alloc] initWithIndex:806 + score:0.997122f + categoryName:@"soccer ball" + displayName:nil] ]; + + MPPImage *image = [self imageWithFileInfo:kMultiObjectsImage]; + + // roi around soccer ball + MPPImageClassifierResult *imageClassifierResult = + [imageClassifier classifyImage:image + regionOfInterest:CGRectMake(0.450f, 0.308f, 0.164f, 0.426f) + error:nil]; + [self assertImageClassifierResult:imageClassifierResult + hasExpectedCategoriesCount:maxResults + expectedCategories:expectedCategories]; +} + +- (void)testClassifyWithRotationSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + NSInteger maxResults = 3; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = @[ + [[MPPCategory alloc] initWithIndex:934 + score:0.622074f + categoryName:@"cheeseburger" + displayName:nil], + [[MPPCategory alloc] initWithIndex:963 + score:0.051214f + categoryName:@"meat loaf" + displayName:nil], + [[MPPCategory alloc] initWithIndex:925 + score:0.048719f + categoryName:@"guacamole" + displayName:nil] + + ]; + + MPPImage *image = [self imageWithFileInfo:kBurgerRotatedImage orientation:UIImageOrientationRight]; + + [self assertResultsOfClassifyImage:image + usingImageClassifier:imageClassifier + expectedCategoriesCount:maxResults + equalsCategories:expectedCategories]; +} + +- (void)testClassifyWithRegionOfInterestAndRotationSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + NSInteger maxResults = 1; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + NSArray *expectedCategories = + @[ [[MPPCategory alloc] initWithIndex:560 + score:0.682305f + categoryName:@"folding chair" + displayName:nil] ]; + + MPPImage *image = [self imageWithFileInfo:kMultiObjectsRotatedImage orientation:UIImageOrientationRight]; + + // roi around folding chair + MPPImageClassifierResult *imageClassifierResult = + [imageClassifier classifyImage:image + regionOfInterest:CGRectMake(0.0f, 0.1763f, 0.5642f, 0.1286f) + error:nil]; + [self assertImageClassifierResult:imageClassifierResult + hasExpectedCategoriesCount:maxResults + expectedCategories:expectedCategories]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateImageClassifierFailsWithResultListenerInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.runningMode = runningModesToTest[i]; + options.completion = ^(MPPImageClassifierResult *result, NSError *error) { + }; + + [self + assertCreateImageClassifierWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode, a " + @"user-defined result callback should not be provided." + }]]; + } +} + +- (void)testCreateImageClassifierFailsWithMissingResultListenerInLiveStreamMode { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateImageClassifierWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode, a " + @"user-defined result callback must be provided." + }]]; +} + +- (void)testClassifyFailsWithCallingWrongApiInImageMode { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([imageClassifier classifyAsyncImage:image + timestampMs:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([imageClassifier classifyVideoFrame:image timestampMs:0 error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testClassifyFailsWithCallingWrongApiInVideoMode { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.runningMode = MPPRunningModeVideo; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([imageClassifier classifyAsyncImage:image + timestampMs:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([imageClassifier classifyImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testClassifyFailsWithCallingWrongApiInLiveStreamMode { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.runningMode = MPPRunningModeLiveStream; + options.completion = ^(MPPImageClassifierResult *result, NSError *error) { + + }; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + NSError *imageApiCallError; + XCTAssertFalse([imageClassifier classifyImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([imageClassifier classifyVideoFrame:image timestampMs:0 error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testClassifyWithVideoModeSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + options.runningMode = MPPRunningModeVideo; + + NSInteger maxResults = 3; + options.maxResults = maxResults; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + for (int i = 0; i < 3; i++) { + MPPImageClassifierResult *imageClassifierResult = [imageClassifier classifyVideoFrame:image + timestampMs:i + error:nil]; + [self assertImageClassifierResult:imageClassifierResult + hasExpectedCategoriesCount:maxResults + expectedCategories:[MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + } +} + +- (void)testClassifyWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + NSInteger maxResults = 3; + options.maxResults = maxResults; + + options.runningMode = MPPRunningModeLiveStream; + options.completion = ^(MPPImageClassifierResult *result, NSError *error) { + [self assertImageClassifierResult:result + hasExpectedCategoriesCount:maxResults + expectedCategories:[MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + }; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + XCTAssertTrue([imageClassifier classifyAsyncImage:image timestampMs:1 error:nil]); + + NSError *error; + XCTAssertFalse([imageClassifier classifyAsyncImage:image timestampMs:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); +} + +- (void)testClassifyWithLiveStreamModeSucceeds { + MPPImageClassifierOptions *options = [self imageClassifierOptionsWithModelName:kFloatModelName]; + + NSInteger maxResults = 3; + options.maxResults = maxResults; + + options.runningMode = MPPRunningModeLiveStream; + options.completion = ^(MPPImageClassifierResult *result, NSError *error) { + [self assertImageClassifierResult:result + hasExpectedCategoriesCount:maxResults + expectedCategories:[MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + }; + + MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; + + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used with the iOS camera. + // AVCaptureVideoDataOutput sample buffer delegates provide frames of type `CMSampleBuffer`. + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + + + for (int i = 0; i < 3; i++) { + XCTAssertTrue([imageClassifier classifyAsyncImage:image timestampMs:i error:nil]); + } +} + +@end From 55483776ab8a990a6ea312286b181502e6862b42 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:44:56 +0530 Subject: [PATCH 11/18] Fixed Issue with Flow Limiter in MPPTaskInfo --- mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 80ff594a..2c078504 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -117,8 +117,8 @@ using ::mediapipe::InputStreamInfo; flowLimitCalculatorNode->add_output_stream(strippedTaskInputStream.cppString); } - NSString *firstOutputStream = self.outputStreams[0]; - auto finishedOutputStream = "FINISHED:" + firstOutputStream.cppString; + NSString *strippedFirstOutputStream = [MPPTaskInfo stripTagIndex:self.outputStreams[0]]; + auto finishedOutputStream = "FINISHED:" + strippedFirstOutputStream.cppString; flowLimitCalculatorNode->add_input_stream(finishedOutputStream); return graphConfig; From 1685664bdbfe2b6df7b3eba28aa6a5365a48c0ee Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:45:53 +0530 Subject: [PATCH 12/18] Fixed formatting --- mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 2c078504..5f614632 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -106,13 +106,13 @@ using ::mediapipe::InputStreamInfo; for (NSString *inputStream in self.inputStreams) { graphConfig.add_input_stream(inputStream.cppString); + + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; + taskSubgraphNode->add_input_stream(taskInputStream.cppString); NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; flowLimitCalculatorNode->add_input_stream(strippedInputStream.cppString); - NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; - taskSubgraphNode->add_input_stream(taskInputStream.cppString); - NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; flowLimitCalculatorNode->add_output_stream(strippedTaskInputStream.cppString); } From aa760855ee5dbf2458306a4be9a2b4b3629c5c77 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:48:50 +0530 Subject: [PATCH 13/18] Updated formatting --- mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 5f614632..63769472 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -106,7 +106,7 @@ using ::mediapipe::InputStreamInfo; for (NSString *inputStream in self.inputStreams) { graphConfig.add_input_stream(inputStream.cppString); - + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; taskSubgraphNode->add_input_stream(taskInputStream.cppString); From 59e0b1ba7454b57599fde07c0fe7d145b3366887 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:51:25 +0530 Subject: [PATCH 14/18] Added stream info for some modes in MPPImageClassifier --- .../tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h | 1 + .../tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm | 5 +++++ .../utils/sources/MPPImageClassifierOptions+Helpers.mm | 5 +++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h index d52df2ae..d1810d56 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h @@ -20,6 +20,7 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPBaseOptions (Helpers) - (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto; +- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto withUseStreamMode:(BOOL)useStreamMode; @end diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index 42cafe61..f7f8e5a5 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -22,6 +22,11 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; @implementation MPPBaseOptions (Helpers) +- (void)copyToProto:(BaseOptionsProto *)baseOptionsProto withUseStreamMode:(BOOL)useStreamMode { + [self copyToProto:baseOptionsProto]; + baseOptionsProto->set_use_stream_mode(useStreamMode); +} + - (void)copyToProto:(BaseOptionsProto *)baseOptionsProto { baseOptionsProto->Clear(); diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm index 36ecf909..926099c1 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm @@ -32,8 +32,9 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto - (void)copyToProto:(CalculatorOptionsProto *)optionsProto { ImageClassifierGraphOptionsProto *graphOptions = optionsProto->MutableExtension(ImageClassifierGraphOptionsProto::ext); - [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; - + + [self.baseOptions copyToProto:graphOptions->mutable_base_options() withUseStreamMode:self.runningMode != MPPRunningModeImage]; + ClassifierOptionsProto *classifierOptionsProto = graphOptions->mutable_classifier_options(); classifierOptionsProto->Clear(); From f51736e32d82bd41e68fb712402c7d7853543df4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:52:14 +0530 Subject: [PATCH 15/18] Added flow limiting for live stream mode in MPPImageClassifier --- .../ios/vision/image_classifier/sources/MPPImageClassifier.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index b817603f..d093acc2 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -69,7 +69,7 @@ static NSString *const kTaskGraphName = [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName] ] taskOptions:options - enableFlowLimiting:NO + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream error:error]; if (!taskInfo) { From da8b60700bca831c8f3e9168e278a6777ab9bd0e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:53:35 +0530 Subject: [PATCH 16/18] Added flow limiter calculator to iOS vision tasks --- mediapipe/tasks/ios/vision/core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 1efb3c33..15c4fa2b 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -55,6 +55,7 @@ objc_library( deps = [ ":MPPRunningMode", "@ios_opencv//:OpencvFramework", + "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", From 8682a3ffd989d0214e92ca161777d6c6696e44cb Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 19:58:03 +0530 Subject: [PATCH 17/18] Updated formatted --- .../utils/sources/MPPBaseOptions+Helpers.h | 3 +- .../MPPImageClassifierTests.m | 100 ++++++++++-------- .../vision/utils/sources/MPPImage+TestUtils.h | 11 +- .../vision/utils/sources/MPPImage+TestUtils.m | 19 ++-- .../core/sources/MPPVisionTaskRunner.mm | 2 +- 5 files changed, 75 insertions(+), 60 deletions(-) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h index d1810d56..408d1a3e 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h @@ -20,7 +20,8 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPBaseOptions (Helpers) - (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto; -- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto withUseStreamMode:(BOOL)useStreamMode; +- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto + withUseStreamMode:(BOOL)useStreamMode; @end diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m index 6be32575..e50fb120 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -23,7 +23,8 @@ static NSString *const kQuantizedModelName = @"mobilenet_v1_0.25_224_quant"; static NSDictionary *const kBurgerImage = @{@"name" : @"burger", @"type" : @"jpg"}; static NSDictionary *const kBurgerRotatedImage = @{@"name" : @"burger_rotated", @"type" : @"jpg"}; static NSDictionary *const kMultiObjectsImage = @{@"name" : @"multi_objects", @"type" : @"jpg"}; -static NSDictionary *const kMultiObjectsRotatedImage = @{@"name" : @"multi_objects_rotated", @"type" : @"jpg"}; +static NSDictionary *const kMultiObjectsRotatedImage = + @{@"name" : @"multi_objects_rotated", @"type" : @"jpg"}; static const int kMobileNetCategoriesCount = 1001; static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; @@ -112,7 +113,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; #pragma mark Assert Classify Results --(MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPImageClassifierTests class] fileName:fileInfo[@"name"] ofType:fileInfo[@"type"]]; @@ -121,11 +122,12 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; return image; } --(MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo orientation:(UIImageOrientation)orientation { +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo + orientation:(UIImageOrientation)orientation { MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPImageClassifierTests class] fileName:fileInfo[@"name"] ofType:fileInfo[@"type"] - orientation:orientation]; + orientation:orientation]; XCTAssertNotNil(image); return image; @@ -173,9 +175,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; } - (void)assertResultsOfClassifyImageWithFileInfo:(NSDictionary *)fileInfo - usingImageClassifier:(MPPImageClassifier *)imageClassifier - expectedCategoriesCount:(NSInteger)expectedCategoriesCount - equalsCategories:(NSArray *)expectedCategories { + usingImageClassifier:(MPPImageClassifier *)imageClassifier + expectedCategoriesCount:(NSInteger)expectedCategoriesCount + equalsCategories:(NSArray *)expectedCategories { MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; [self assertResultsOfClassifyImage:mppImage @@ -242,11 +244,13 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; - (void)testClassifyWithModelPathAndFloatModelSucceeds { MPPImageClassifier *imageClassifier = [self imageClassifierFromModelFileWithName:kFloatModelName]; - [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:kMobileNetCategoriesCount - equalsCategories:[MPPImageClassifierTests - expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + [self + assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:kMobileNetCategoriesCount + equalsCategories: + [MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; } - (void)testClassifyWithOptionsAndFloatModelSucceeds { @@ -257,11 +261,13 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; - [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:maxResults - equalsCategories:[MPPImageClassifierTests - expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + [self + assertResultsOfClassifyImageWithFileInfo:kBurgerImage + usingImageClassifier:imageClassifier + expectedCategoriesCount:maxResults + equalsCategories: + [MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; } - (void)testClassifyWithQuantizedModelSucceeds { @@ -279,9 +285,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; displayName:nil] ]; [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:maxResults - equalsCategories:expectedCategories]; + usingImageClassifier:imageClassifier + expectedCategoriesCount:maxResults + equalsCategories:expectedCategories]; } - (void)testClassifyWithScoreThresholdSucceeds { @@ -297,9 +303,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; displayName:nil] ]; [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:expectedCategories.count - equalsCategories:expectedCategories]; + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; } - (void)testClassifyWithAllowListSucceeds { @@ -326,9 +332,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; ]; [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:expectedCategories.count - equalsCategories:expectedCategories]; + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; } - (void)testClassifyWithDenyListSucceeds { @@ -358,9 +364,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; ]; [self assertResultsOfClassifyImageWithFileInfo:kBurgerImage - usingImageClassifier:imageClassifier - expectedCategoriesCount:expectedCategories.count - equalsCategories:expectedCategories]; + usingImageClassifier:imageClassifier + expectedCategoriesCount:expectedCategories.count + equalsCategories:expectedCategories]; } - (void)testClassifyWithRegionOfInterestSucceeds { @@ -377,7 +383,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; displayName:nil] ]; MPPImage *image = [self imageWithFileInfo:kMultiObjectsImage]; - + // roi around soccer ball MPPImageClassifierResult *imageClassifierResult = [imageClassifier classifyImage:image @@ -412,7 +418,8 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; ]; - MPPImage *image = [self imageWithFileInfo:kBurgerRotatedImage orientation:UIImageOrientationRight]; + MPPImage *image = [self imageWithFileInfo:kBurgerRotatedImage + orientation:UIImageOrientationRight]; [self assertResultsOfClassifyImage:image usingImageClassifier:imageClassifier @@ -434,7 +441,8 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; categoryName:@"folding chair" displayName:nil] ]; - MPPImage *image = [self imageWithFileInfo:kMultiObjectsRotatedImage orientation:UIImageOrientationRight]; + MPPImage *image = [self imageWithFileInfo:kMultiObjectsRotatedImage + orientation:UIImageOrientationRight]; // roi around folding chair MPPImageClassifierResult *imageClassifierResult = @@ -606,8 +614,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; - - MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; for (int i = 0; i < 3; i++) { MPPImageClassifierResult *imageClassifierResult = [imageClassifier classifyVideoFrame:image @@ -615,8 +622,9 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; error:nil]; [self assertImageClassifierResult:imageClassifierResult hasExpectedCategoriesCount:maxResults - expectedCategories:[MPPImageClassifierTests - expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + expectedCategories: + [MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; } } @@ -630,13 +638,14 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; options.completion = ^(MPPImageClassifierResult *result, NSError *error) { [self assertImageClassifierResult:result hasExpectedCategoriesCount:maxResults - expectedCategories:[MPPImageClassifierTests - expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + expectedCategories: + [MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; }; MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; - MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; XCTAssertTrue([imageClassifier classifyAsyncImage:image timestampMs:1 error:nil]); @@ -663,16 +672,17 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; options.completion = ^(MPPImageClassifierResult *result, NSError *error) { [self assertImageClassifierResult:result hasExpectedCategoriesCount:maxResults - expectedCategories:[MPPImageClassifierTests - expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; + expectedCategories: + [MPPImageClassifierTests + expectedResultCategoriesForClassifyBurgerImageWithFloatModel]]; }; MPPImageClassifier *imageClassifier = [self imageClassifierWithOptionsSucceeds:options]; - - // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used with the iOS camera. - // AVCaptureVideoDataOutput sample buffer delegates provide frames of type `CMSampleBuffer`. - MPPImage *image = [self imageWithFileInfo:kBurgerImage]; + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used + // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type + // `CMSampleBuffer`. + MPPImage *image = [self imageWithFileInfo:kBurgerImage]; for (int i = 0; i < 3; i++) { XCTAssertTrue([imageClassifier classifyAsyncImage:image timestampMs:i error:nil]); diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h index eb0edf6a..9b45b259 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h @@ -40,7 +40,8 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(imageFromBundle(class:filename:type:)); /** - * Loads an image from a file in an app bundle into a `MPPImage` object with the specified orientation. + * Loads an image from a file in an app bundle into a `MPPImage` object with the specified + * orientation. * * @param classObject The specified class associated with the bundle containing * the file to be loaded. @@ -52,10 +53,10 @@ NS_ASSUME_NONNULL_BEGIN * nil if it cannot load the image. */ + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject - fileName:(NSString *)name - ofType:(NSString *)type - orientation:(UIImageOrientation)imageOrientation - NS_SWIFT_NAME(imageFromBundle(class:filename:type:orientation:)); + fileName:(NSString *)name + ofType:(NSString *)type + orientation:(UIImageOrientation)imageOrientation + NS_SWIFT_NAME(imageFromBundle(class:filename:type:orientation:)); @end diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m index 5421fd3b..0b0ef9fb 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m @@ -16,14 +16,17 @@ @interface UIImage (FileUtils) -+(nullable UIImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type; ++ (nullable UIImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type; @end @implementation UIImage (FileUtils) -+(nullable UIImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type { - ++ (nullable UIImage *)imageFromBundleWithClass:(Class)classObject + fileName:(NSString *)name + ofType:(NSString *)type { NSString *imagePath = [[NSBundle bundleForClass:classObject] pathForResource:name ofType:type]; if (!imagePath) return nil; @@ -35,17 +38,17 @@ @implementation MPPImage (TestUtils) + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject - fileName:(NSString *)name - ofType:(NSString *)type { + fileName:(NSString *)name + ofType:(NSString *)type { UIImage *image = [UIImage imageFromBundleWithClass:classObject fileName:name ofType:type]; return [[MPPImage alloc] initWithUIImage:image error:nil]; } + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject - fileName:(NSString *)name - ofType:(NSString *)type - orientation:(UIImageOrientation)imageOrientation { + fileName:(NSString *)name + ofType:(NSString *)type + orientation:(UIImageOrientation)imageOrientation { UIImage *image = [UIImage imageFromBundleWithClass:classObject fileName:name ofType:type]; return [[MPPImage alloc] initWithUIImage:image orientation:imageOrientation error:nil]; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 787dceea..0ed447c0 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -138,7 +138,7 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; } - (std::optional)processImagePacketMap:(const PacketMap &)packetMap - error:(NSError **)error { + error:(NSError **)error { if (_runningMode != MPPRunningModeImage) { [MPPCommonUtils createCustomError:error From 58adb69c44d147cc7d48eb59a74216cc8c01bb0e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 23 Mar 2023 20:16:30 +0530 Subject: [PATCH 18/18] Removed unwanted iOS tests --- .../image_classifier/MPPImageClassifierTests.m | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m index e50fb120..e09647b0 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -224,23 +224,6 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; }]]; } -// - (void)testCreateImageClassifierFailsWithInvalidMaxResults { -// MPPImageClassifierOptions *options = -// [self imageClassifierOptionsWithModelName:kFloatModelName]; -// options.maxResults = 0; - -// [self assertCreateImageClassifierWithOptions:options -// failsWithExpectedError: -// [NSError errorWithDomain:kExpectedErrorDomain -// code:MPPTasksErrorCodeInvalidArgumentError -// userInfo:@{ -// NSLocalizedDescriptionKey : -// @"INVALID_ARGUMENT: Invalid `max_results` -// option: " -// @"value must be > 0." -// }]]; -// } - - (void)testClassifyWithModelPathAndFloatModelSucceeds { MPPImageClassifier *imageClassifier = [self imageClassifierFromModelFileWithName:kFloatModelName];