From e6fd39b3eed179c031d1825c48e93610f60eea7d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 25 May 2023 20:43:51 +0530 Subject: [PATCH] Updated the vision task runner to split the method that creates normalized rect based on ROI --- .../vision/core/sources/MPPVisionTaskRunner.h | 51 +++++++++++++------ .../core/sources/MPPVisionTaskRunner.mm | 25 ++++++++- .../sources/MPPImageClassifier.mm | 18 +++---- .../sources/MPPObjectDetector.mm | 8 +-- 4 files changed, 69 insertions(+), 33 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 318b2405..a7930244 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -58,38 +58,57 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * Creates a `NormalizedRect` from a region of interest and an image orientation, performing - * sanity checks on-the-fly. - * If the input region of interest equals `CGRectZero`, returns a default `NormalizedRect` covering - * the whole image with rotation set according `imageOrientation`. If `ROIAllowed` is NO, an error - * will be returned if the input region of interest is not equal to `CGRectZero`. Mirrored - * orientations (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`, + * Creates a `NormalizedRect` from image orientation for a task which does not support roi, + * performing sanity checks on-the-fly. Mirrored orientations + * (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`, * `UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`) are not supported. An error * will be returned if `imageOrientation` is equal to any one of them. * - * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals - * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals - * `CGRectZero` if `ROIAllowed` is NO. Otherwise, an error will be returned. - * @param imageSize A `CGSize` specifying the size of the image within which normalized rect is - * calculated. * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. * Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`, * `UIImageOrientationLeftMirrored`, `UIImageOrientationRightMirrored`) are not supported. An error * will be returned if `imageOrientation` is equal to any one of them. - * @param ROIAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. + * @param imageSize A `CGSize` specifying the size of the image within which normalized rect is + * calculated. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An optional `NormalizedRect` from the given region of interest and image orientation. + */ +- (std::optional)normalizedRectWithImageOrientation: + (UIImageOrientation)imageOrientation + imageSize:(CGSize)imageSize + error:(NSError **)error; + +/** + * Creates a `NormalizedRect` from roi and image orientation for a task which supports roi, + * performing sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns + * a default `NormalizedRect` covering the whole image with rotation set according + * `imageOrientation`. Mirrored orientations + * (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`, + * `UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`) are not supported. An error + * will be returned if `imageOrientation` is equal to any one of them. + * + * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals + * `CGRectZero`, the returned `NormalizedRect` covers the whole image. + * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the + * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. + * Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`, + * `UIImageOrientationLeftMirrored`, `UIImageOrientationRightMirrored`) are not supported. An error + * will be returned if `imageOrientation` is equal to any one of them. + * @param imageSize A `CGSize` specifying the size of the image within which normalized rect is + * calculated. * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no * error will be saved. * * @return An optional `NormalizedRect` from the given region of interest and image orientation. */ - (std::optional) - normalizedRectFromRegionOfInterest:(CGRect)roi - imageSize:(CGSize)imageSize + normalizedRectWithRegionOfInterest:(CGRect)roi imageOrientation:(UIImageOrientation)imageOrientation - ROIAllowed:(BOOL)ROIAllowed + imageSize:(CGSize)imageSize error:(NSError **)error; - /** * A synchronous method to invoke the C++ task runner to process single image inputs. The call * blocks the current thread until a failure status or a successful result is returned. diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 0089e516..c1b5d058 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -91,7 +91,30 @@ static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision"; return self; } -- (std::optional)normalizedRectFromRegionOfInterest:(CGRect)roi +- (std::optional)normalizedRectWithRegionOfInterest:(CGRect)roi + imageOrientation: + (UIImageOrientation)imageOrientation + imageSize:(CGSize)imageSize + error:(NSError **)error { + return [self normalizedRectWithRegionOfInterest:roi + imageSize:imageSize + imageOrientation:imageOrientation + ROIAllowed:YES + error:error]; +} + +- (std::optional)normalizedRectWithImageOrientation: + (UIImageOrientation)imageOrientation + imageSize:(CGSize)imageSize + error:(NSError **)error { + return [self normalizedRectWithRegionOfInterest:CGRectZero + imageSize:imageSize + imageOrientation:imageOrientation + ROIAllowed:NO + error:error]; +} + +- (std::optional)normalizedRectWithRegionOfInterest:(CGRect)roi imageSize:(CGSize)imageSize imageOrientation: (UIImageOrientation)imageOrientation diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 3ad8d0de..10f6fc26 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -166,10 +166,9 @@ static const int kMicroSecondsPerMilliSecond = 1000; regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi - imageSize:CGSizeMake(image.width, image.height) + [_visionTaskRunner normalizedRectWithRegionOfInterest:roi imageOrientation:image.orientation - ROIAllowed:YES + imageSize:CGSizeMake(image.width, image.height) error:error]; if (!rect.has_value()) { return nil; @@ -196,15 +195,18 @@ static const int kMicroSecondsPerMilliSecond = 1000; outputPacketMap.value()[kClassificationsStreamName.cppString]]; } +- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error { + return [self classifyImage:image regionOfInterest:CGRectZero error:error]; +} + - (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi - imageSize:CGSizeMake(image.width, image.height) + [_visionTaskRunner normalizedRectWithRegionOfInterest:roi imageOrientation:image.orientation - ROIAllowed:YES + imageSize:CGSizeMake(image.width, image.height) error:error]; if (!rect.has_value()) { return std::nullopt; @@ -225,10 +227,6 @@ static const int kMicroSecondsPerMilliSecond = 1000; return inputPacketMap; } -- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error { - return [self classifyImage:image regionOfInterest:CGRectZero error:error]; -} - - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds regionOfInterest:(CGRect)roi diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 27b196d7..b7924996 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -160,10 +160,8 @@ static NSString *const kTaskName = @"objectDetector"; timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation imageSize:CGSizeMake(image.width, image.height) - imageOrientation:image.orientation - ROIAllowed:NO error:error]; if (!rect.has_value()) { return std::nullopt; @@ -188,10 +186,8 @@ static NSString *const kTaskName = @"objectDetector"; regionOfInterest:(CGRect)roi error:(NSError **)error { std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation imageSize:CGSizeMake(image.width, image.height) - imageOrientation:image.orientation - ROIAllowed:NO error:error]; if (!rect.has_value()) { return nil;