From dc393b0bd42316926712d662589942d2b629be7c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:43:01 +0530 Subject: [PATCH] Added methods to MPPVisionTaskRunner --- .../vision/core/sources/MPPVisionTaskRunner.h | 77 ++++++++++++++- .../core/sources/MPPVisionTaskRunner.mm | 93 +++++++++++++++++++ 2 files changed, 166 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 84b65730..5212f65f 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -13,10 +13,13 @@ // limitations under the License. #import +#import #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#include "mediapipe/framework/formats/rect.pb.h" + NS_ASSUME_NONNULL_BEGIN /** @@ -54,10 +57,76 @@ NS_ASSUME_NONNULL_BEGIN (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; -- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig - packetsCallback: - (mediapipe::tasks::core::PacketsCallback)packetsCallback - error:(NSError **)error NS_UNAVAILABLE; +/** + * Creates a `NormalizedRect` from region of interest and image orientation, performing + * sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns a default + * `NormalizedRect` covering the whole image with rotation set according `imageOrientation`. If + * `roiAllowed` is NO, an error will be returned if the input region of interest is not equal to + * `CGRectZero`. + * + * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals + * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals + * `CGRectZero` if `roiAllowed` is NO. Otherwise an error will be returned. + * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the + * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. + * @param roiAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. + * + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `NormalizedRect` from the given region of interest and image orientation. + */ +- (std::optional) + normalizedRectFromRegionOfInterest:(CGRect)roi + imageOrientation:(UIImageOrientation)imageOrientation + roiAllowed:(BOOL)roiAllowed + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process single image inputs. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional) + processImagePacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process continuous video frames. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional) + processVideoFramePacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; + +/** + * An asynchronous method to send live stream data to the C++ task runner. The call blocks the + * current thread until a failure status or a successful result is returned. The results will be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. + * Please note that any errors during processing of the live stream packet will only be available in + * the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + */ +- (BOOL)processLiveStreamPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index bfa9e34e..3c115f1c 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -17,8 +17,14 @@ #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#include "absl/status/statusor.h" + +#include + namespace { using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::NormalizedRect; +using ::mediapipe::tasks::core::PacketMap; using ::mediapipe::tasks::core::PacketsCallback; } // namespace @@ -70,4 +76,91 @@ using ::mediapipe::tasks::core::PacketsCallback; return self; } +- (std::optional)normalizedRectFromRegionOfInterest:(CGRect)roi + imageOrientation: + (UIImageOrientation)imageOrientation + roiAllowed:(BOOL)roiAllowed + error:(NSError **)error { + if (CGRectEqualToRect(roi, CGRectZero) && !roiAllowed) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"This task doesn't support region-of-interest."]; + return std::nullopt; + } + + CGRect calculatedRoi = CGRectEqualToRect(roi, CGRectZero) ? roi : CGRectMake(0.0, 0.0, 1.0, 1.0); + + NormalizedRect normalizedRect; + normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi)); + normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi)); + normalizedRect.set_width(CGRectGetWidth(calculatedRoi)); + normalizedRect.set_height(CGRectGetHeight(calculatedRoi)); + + int rotationDegrees = 0; + switch (imageOrientation) { + case UIImageOrientationUp: + break; + case UIImageOrientationRight: { + rotationDegrees = -90; + break; + } + case UIImageOrientationDown: { + rotationDegrees = -180; + break; + } + case UIImageOrientationLeft: { + rotationDegrees = -270; + break; + } + default: + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unsupported UIImageOrientation."]; + } + + normalizedRect.set_rotation(-rotationDegrees * M_PI / 180.0); + + return normalizedRect; +} + +- (std::optional)processImagePacketMap:(PacketMap &)packetMap error:(NSError **)error { + if (_runningMode != MPPRunningModeImage) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return std::nullopt; + } + + return [self processPacketMap:packetMap error:error]; +} + +- (std::optional)processVideoFramePacketMap:(PacketMap &)packetMap + error:(NSError **)error { + if (_runningMode != MPPRunningModeVideo) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return std::nullopt; + } + + return [self processPacketMap:packetMap error:error]; +} + +- (BOOL)processLiveStreamPacketMap:(PacketMap &)packetMap error:(NSError **)error { + if (_runningMode != MPPRunningModeLiveStream) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return NO; + } + + return [self sendPacketMap:packetMap error:error]; +} + @end