From 87ba86ace2ccf0a74ae2629a373fb52f6e96787e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:34:44 +0530 Subject: [PATCH 01/25] Added method to send packet map to C++ task runner --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 12 ++++++++++++ mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm | 3 +++ 2 files changed, 15 insertions(+) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 704fc453..fbb12295 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -74,6 +74,18 @@ NS_ASSUME_NONNULL_BEGIN */ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; +- (std::optional) + processPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; + +/** + * An asynchronous method that is designed for handling live streaming data such as live camera. A + * user-defined PacketsCallback function must be provided in the constructor to receive the output + * packets. The caller must ensure that the input packet timestamps are monotonically increasing. + * This method is thread-unsafe and it is the caller's responsibility to synchronize access to this + * method across multiple threads and to ensure that the input packet timestamps are in order. + */ +- (BOOL)sendPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap error:(NSError **)error; /** * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index eb777679..f7db9553 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -52,6 +52,9 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; - (absl::StatusOr)process:(const PacketMap &)packetMap { return _cppTaskRunner->Process(packetMap); +- (BOOL)sendPacketMap:(const PacketMap &)packetMap error:(NSError **)error { + absl::Status sendStatus = _cppTaskRunner->Send(packetMap); + return [MPPCommonUtils checkCppError:sendStatus toError:error]; } - (absl::Status)close { From 6d7f172e9fdd77b44db0a1228c32cf95fa9d693d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:35:05 +0530 Subject: [PATCH 02/25] Changed return type of process method in MPPTaskRunner --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 4 ++-- mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index fbb12295..b2881ac0 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -17,6 +17,8 @@ #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/tasks/cc/core/task_runner.h" +#include + NS_ASSUME_NONNULL_BEGIN /** @@ -72,8 +74,6 @@ NS_ASSUME_NONNULL_BEGIN * caller's responsibility to synchronize access to this method across multiple threads and to * ensure that the input packet timestamps are in order. */ -- (absl::StatusOr)process: - (const mediapipe::tasks::core::PacketMap &)packetMap; - (std::optional) processPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap error:(NSError **)error; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index f7db9553..a5784633 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -50,8 +50,14 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; return self; } -- (absl::StatusOr)process:(const PacketMap &)packetMap { - return _cppTaskRunner->Process(packetMap); +- (std::optional)processPacketMap:(const PacketMap &)packetMap error:(NSError **)error { + absl::StatusOr resultPacketMap = _cppTaskRunner->Process(packetMap); + if (![MPPCommonUtils checkCppError:resultPacketMap.status() toError:error]) { + return std::nullopt; + } + return resultPacketMap.value(); +} + - (BOOL)sendPacketMap:(const PacketMap &)packetMap error:(NSError **)error { absl::Status sendStatus = _cppTaskRunner->Send(packetMap); return [MPPCommonUtils checkCppError:sendStatus toError:error]; From 045050fc85d11ee3b9f35591708dee83060822fb Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:36:34 +0530 Subject: [PATCH 03/25] Changed method Updated method calls to process packet map in iOS text tasks --- .../text/text_classifier/sources/MPPTextClassifier.mm | 11 ++++++----- .../ios/text/text_embedder/sources/MPPTextEmbedder.mm | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm index 52e4d92a..871df0f8 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -83,15 +83,16 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_classifier.T Packet packet = [MPPTextPacketCreator createWithText:text]; std::map packetMap = {{kTextInStreamName.cppString, packet}}; - absl::StatusOr statusOrOutputPacketMap = [_textTaskRunner process:packetMap]; + std::optional outputPacketMap = [_textTaskRunner processPacketMap:packetMap + error:error]; - if (![MPPCommonUtils checkCppError:statusOrOutputPacketMap.status() toError:error]) { + if (!outputPacketMap.has_value()) { return nil; } - return [MPPTextClassifierResult - textClassifierResultWithClassificationsPacket:statusOrOutputPacketMap.value() - [kClassificationsStreamName.cppString]]; + return + [MPPTextClassifierResult textClassifierResultWithClassificationsPacket: + outputPacketMap.value()[kClassificationsStreamName.cppString]]; } @end diff --git a/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm b/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm index 62eb882d..80a34f0c 100644 --- a/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm +++ b/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm @@ -83,14 +83,15 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_embedder.Tex Packet packet = [MPPTextPacketCreator createWithText:text]; std::map packetMap = {{kTextInStreamName.cppString, packet}}; - absl::StatusOr statusOrOutputPacketMap = [_textTaskRunner process:packetMap]; - if (![MPPCommonUtils checkCppError:statusOrOutputPacketMap.status() toError:error]) { + std::optional outputPacketMap = [_textTaskRunner processPacketMap:packetMap + error:error]; + + if (!outputPacketMap.has_value()) { return nil; } - return [MPPTextEmbedderResult - textEmbedderResultWithOutputPacket:statusOrOutputPacketMap + textEmbedderResultWithOutputPacket:outputPacketMap .value()[kEmbeddingsOutStreamName.cppString]]; } From b76ab3739482244e41e9757467fa5277efbd622e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:37:24 +0530 Subject: [PATCH 04/25] Removed unwanted imports --- mediapipe/tasks/ios/text/text_classifier/BUILD | 1 - .../tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm | 1 - mediapipe/tasks/ios/text/text_embedder/BUILD | 1 - .../tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm | 2 -- 4 files changed, 5 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD index c56d51e5..7913340a 100644 --- a/mediapipe/tasks/ios/text/text_classifier/BUILD +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -58,6 +58,5 @@ objc_library( "//mediapipe/tasks/ios/text/core:MPPTextTaskRunner", "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierOptionsHelpers", "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierResultHelpers", - "@com_google_absl//absl/status:statusor", ], ) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm index 871df0f8..f0e1e415 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -22,7 +22,6 @@ #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" -#include "absl/status/statusor.h" #include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" namespace { diff --git a/mediapipe/tasks/ios/text/text_embedder/BUILD b/mediapipe/tasks/ios/text/text_embedder/BUILD index 74aefdf7..a600d536 100644 --- a/mediapipe/tasks/ios/text/text_embedder/BUILD +++ b/mediapipe/tasks/ios/text/text_embedder/BUILD @@ -58,6 +58,5 @@ objc_library( "//mediapipe/tasks/ios/text/core:MPPTextTaskRunner", "//mediapipe/tasks/ios/text/text_embedder/utils:MPPTextEmbedderOptionsHelpers", "//mediapipe/tasks/ios/text/text_embedder/utils:MPPTextEmbedderResultHelpers", - "@com_google_absl//absl/status:statusor", ], ) diff --git a/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm b/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm index 80a34f0c..e0f0d549 100644 --- a/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm +++ b/mediapipe/tasks/ios/text/text_embedder/sources/MPPTextEmbedder.mm @@ -23,8 +23,6 @@ #import "mediapipe/tasks/ios/text/text_embedder/utils/sources/MPPTextEmbedderOptions+Helpers.h" #import "mediapipe/tasks/ios/text/text_embedder/utils/sources/MPPTextEmbedderResult+Helpers.h" -#include "absl/status/statusor.h" - namespace { using ::mediapipe::Packet; using ::mediapipe::tasks::core::PacketMap; From dc393b0bd42316926712d662589942d2b629be7c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:43:01 +0530 Subject: [PATCH 05/25] Added methods to MPPVisionTaskRunner --- .../vision/core/sources/MPPVisionTaskRunner.h | 77 ++++++++++++++- .../core/sources/MPPVisionTaskRunner.mm | 93 +++++++++++++++++++ 2 files changed, 166 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 84b65730..5212f65f 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -13,10 +13,13 @@ // limitations under the License. #import +#import #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#include "mediapipe/framework/formats/rect.pb.h" + NS_ASSUME_NONNULL_BEGIN /** @@ -54,10 +57,76 @@ NS_ASSUME_NONNULL_BEGIN (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; -- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig - packetsCallback: - (mediapipe::tasks::core::PacketsCallback)packetsCallback - error:(NSError **)error NS_UNAVAILABLE; +/** + * Creates a `NormalizedRect` from region of interest and image orientation, performing + * sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns a default + * `NormalizedRect` covering the whole image with rotation set according `imageOrientation`. If + * `roiAllowed` is NO, an error will be returned if the input region of interest is not equal to + * `CGRectZero`. + * + * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals + * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals + * `CGRectZero` if `roiAllowed` is NO. Otherwise an error will be returned. + * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the + * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. + * @param roiAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. + * + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `NormalizedRect` from the given region of interest and image orientation. + */ +- (std::optional) + normalizedRectFromRegionOfInterest:(CGRect)roi + imageOrientation:(UIImageOrientation)imageOrientation + roiAllowed:(BOOL)roiAllowed + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process single image inputs. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional) + processImagePacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process continuous video frames. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional) + processVideoFramePacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; + +/** + * An asynchronous method to send live stream data to the C++ task runner. The call blocks the + * current thread until a failure status or a successful result is returned. The results will be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + * + * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. + * Please note that any errors during processing of the live stream packet will only be available in + * the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + */ +- (BOOL)processLiveStreamPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index bfa9e34e..3c115f1c 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -17,8 +17,14 @@ #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#include "absl/status/statusor.h" + +#include + namespace { using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::NormalizedRect; +using ::mediapipe::tasks::core::PacketMap; using ::mediapipe::tasks::core::PacketsCallback; } // namespace @@ -70,4 +76,91 @@ using ::mediapipe::tasks::core::PacketsCallback; return self; } +- (std::optional)normalizedRectFromRegionOfInterest:(CGRect)roi + imageOrientation: + (UIImageOrientation)imageOrientation + roiAllowed:(BOOL)roiAllowed + error:(NSError **)error { + if (CGRectEqualToRect(roi, CGRectZero) && !roiAllowed) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"This task doesn't support region-of-interest."]; + return std::nullopt; + } + + CGRect calculatedRoi = CGRectEqualToRect(roi, CGRectZero) ? roi : CGRectMake(0.0, 0.0, 1.0, 1.0); + + NormalizedRect normalizedRect; + normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi)); + normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi)); + normalizedRect.set_width(CGRectGetWidth(calculatedRoi)); + normalizedRect.set_height(CGRectGetHeight(calculatedRoi)); + + int rotationDegrees = 0; + switch (imageOrientation) { + case UIImageOrientationUp: + break; + case UIImageOrientationRight: { + rotationDegrees = -90; + break; + } + case UIImageOrientationDown: { + rotationDegrees = -180; + break; + } + case UIImageOrientationLeft: { + rotationDegrees = -270; + break; + } + default: + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unsupported UIImageOrientation."]; + } + + normalizedRect.set_rotation(-rotationDegrees * M_PI / 180.0); + + return normalizedRect; +} + +- (std::optional)processImagePacketMap:(PacketMap &)packetMap error:(NSError **)error { + if (_runningMode != MPPRunningModeImage) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return std::nullopt; + } + + return [self processPacketMap:packetMap error:error]; +} + +- (std::optional)processVideoFramePacketMap:(PacketMap &)packetMap + error:(NSError **)error { + if (_runningMode != MPPRunningModeVideo) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return std::nullopt; + } + + return [self processPacketMap:packetMap error:error]; +} + +- (BOOL)processLiveStreamPacketMap:(PacketMap &)packetMap error:(NSError **)error { + if (_runningMode != MPPRunningModeLiveStream) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is not initialized with image mode. Current Running Mode:"]; + return NO; + } + + return [self sendPacketMap:packetMap error:error]; +} + @end From c625d6afdc2fde7cfa85d053596d9ca01d551836 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:43:22 +0530 Subject: [PATCH 06/25] Added methods to MPPVisionPacketCreator --- .../core/sources/MPPVisionPacketCreator.h | 13 +++++++- .../core/sources/MPPVisionPacketCreator.mm | 32 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h index cf597ec2..77dcd197 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h @@ -14,9 +14,11 @@ #import -#include "mediapipe/framework/packet.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/formats/rect.pb.h" + /** * This class helps create various kinds of packets for Mediapipe Vision Tasks. */ @@ -24,4 +26,13 @@ + (mediapipe::Packet)createPacketWithMPPImage:(MPPImage *)image error:(NSError **)error; ++ (mediapipe::Packet)createPacketWithMPPImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error; + ++ (mediapipe::Packet)createPacketWithNormalizedRect:(mediapipe::NormalizedRect &)normalizedRect; + ++ (mediapipe::Packet)createPacketWithNormalizedRect:(mediapipe::NormalizedRect &)normalizedRect + timestampMs:(NSInteger)timestampMs; + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm index 01e583e6..bf136a75 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm @@ -16,18 +16,19 @@ #import "mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h" #include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/timestamp.h" + +static const NSUInteger kMicroSecondsPerMilliSecond = 1000; namespace { using ::mediapipe::Image; using ::mediapipe::ImageFrame; using ::mediapipe::MakePacket; +using ::mediapipe::NormalizedRect; using ::mediapipe::Packet; +using ::mediapipe::Timestamp; } // namespace -struct freeDeleter { - void operator()(void *ptr) { free(ptr); } -}; - @implementation MPPVisionPacketCreator + (Packet)createPacketWithMPPImage:(MPPImage *)image error:(NSError **)error { @@ -40,4 +41,27 @@ struct freeDeleter { return MakePacket(std::move(imageFrame)); } ++ (Packet)createPacketWithMPPImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error { + std::unique_ptr imageFrame = [image imageFrameWithError:error]; + + if (!imageFrame) { + return Packet(); + } + + return MakePacket(std::move(imageFrame)) + .At(Timestamp(int64(timestampMs * kMicroSecondsPerMilliSecond))); +} + ++ (Packet)createPacketWithNormalizedRect:(NormalizedRect &)normalizedRect { + return MakePacket(std::move(normalizedRect)); +} + ++ (Packet)createPacketWithNormalizedRect:(NormalizedRect &)normalizedRect + timestampMs:(NSInteger)timestampMs { + return MakePacket(std::move(normalizedRect)) + .At(Timestamp(int64(timestampMs * kMicroSecondsPerMilliSecond))); +} + @end From c871fc58ec44b6f776e637dd50f6e60121adf026 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:43:46 +0530 Subject: [PATCH 07/25] Updated build targets of vision packet creator and task runner --- mediapipe/tasks/ios/vision/core/BUILD | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 1961ca6b..600ef430 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -26,6 +26,24 @@ objc_library( module_name = "MPPRunningMode", ) +objc_library( + name = "MPPVisionPacketCreator", + srcs = ["sources/MPPVisionPacketCreator.mm"], + hdrs = ["sources/MPPVisionPacketCreator.h"], + sdk_frameworks = ["UIKit"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework:timestamp", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core/utils:MPPImageUtils", + "//mediapipe/framework/formats:rect_cc_proto", + ], +) + objc_library( name = "MPPVisionTaskRunner", srcs = ["sources/MPPVisionTaskRunner.mm"], @@ -39,5 +57,6 @@ objc_library( "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", "//mediapipe/tasks/ios/core:MPPTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", ], ) From abb140ed2ea174c3caa000fb71400f9acbd47ed1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:45:58 +0530 Subject: [PATCH 08/25] Added MPPImageClassifierResultHelpers --- .../ios/vision/image_classifier/utils/BUILD | 29 +++++++++++++ .../MPPImageClassifierResult+Helpers.h | 28 ++++++++++++ .../MPPImageClassifierResult+Helpers.mm | 43 +++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_classifier/utils/BUILD create mode 100644 mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD b/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD new file mode 100644 index 00000000..5effa46f --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD @@ -0,0 +1,29 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageClassifierResultHelpers", + srcs = ["sources/MPPImageClassifierResult+Helpers.mm"], + hdrs = ["sources/MPPImageClassifierResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifierResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h new file mode 100644 index 00000000..2e6921c9 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPImageClassifierResult (Helpers) + ++ (MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: + (const mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm new file mode 100644 index 00000000..e634569b --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +#import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +static const int kMicroSecondsPerMilliSecond = 1000; + +namespace { +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::Packet; +} // namespace + +#define int kMicroSecondsPerMilliSecond = 1000; + +@implementation MPPImageClassifierResult (Helpers) + ++ (MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: + (const Packet &)packet { + MPPClassificationResult *classificationResult = [MPPClassificationResult + classificationResultWithProto:packet.Get()]; + + return [[MPPImageClassifierResult alloc] + initWithClassificationResult:classificationResult + timestampMs:(NSInteger)(packet.Timestamp().Value() / + kMicroSecondsPerMilliSecond)]; +} + +@end From ee6171c8336f631b9e4b0409d8ae31fe8687b31d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:46:11 +0530 Subject: [PATCH 09/25] Added MPPImageClassifierOptionsHelpers --- .../ios/vision/image_classifier/utils/BUILD | 15 +++++ .../MPPImageClassifierOptions+Helpers.h | 27 +++++++++ .../MPPImageClassifierOptions+Helpers.mm | 56 +++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD b/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD index 5effa46f..c1928b6f 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/BUILD @@ -16,6 +16,21 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) +objc_library( + name = "MPPImageClassifierOptionsHelpers", + srcs = ["sources/MPPImageClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPImageClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifierOptions", + ], +) + objc_library( name = "MPPImageClassifierResultHelpers", srcs = ["sources/MPPImageClassifierResult+Helpers.mm"], diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h new file mode 100644 index 00000000..316ca215 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h @@ -0,0 +1,27 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPImageClassifierOptions (Helpers) + +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm new file mode 100644 index 00000000..36ecf909 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.mm @@ -0,0 +1,56 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_classifier/proto/image_classifier_graph_options.pb.h" + +namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using ImageClassifierGraphOptionsProto = + ::mediapipe::tasks::vision::image_classifier::proto::ImageClassifierGraphOptions; +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} // namespace + +@implementation MPPImageClassifierOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + ImageClassifierGraphOptionsProto *graphOptions = + optionsProto->MutableExtension(ImageClassifierGraphOptionsProto::ext); + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + + ClassifierOptionsProto *classifierOptionsProto = graphOptions->mutable_classifier_options(); + classifierOptionsProto->Clear(); + + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + + for (NSString *category in self.categoryAllowlist) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.categoryDenylist) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end From 6eafddb8e232610b24ef366c64402fc615eebac2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:46:20 +0530 Subject: [PATCH 10/25] Added MPPImageClassifier --- .../tasks/ios/vision/image_classifier/BUILD | 26 ++ .../sources/MPPImageClassifier.h | 217 +++++++++++++++++ .../sources/MPPImageClassifier.mm | 228 ++++++++++++++++++ 3 files changed, 471 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h create mode 100644 mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index 45e6e215..130e5fe7 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -36,3 +36,29 @@ objc_library( "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPImageClassifier", + srcs = ["sources/MPPImageClassifier.mm"], + hdrs = ["sources/MPPImageClassifier.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + module_name = "MPPImageClassifier", + deps = [ + ":MPPImageClassifierOptions", + ":MPPImageClassifierResult", + "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", + "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/image_classifier/utils:MPPImageClassifierOptionsHelpers", + "//mediapipe/tasks/ios/vision/image_classifier/utils:MPPImageClassifierResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h new file mode 100644 index 00000000..1914f9ae --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h @@ -0,0 +1,217 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h" +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Performs classification on images. + * + * The API expects a TFLite model with optional, but strongly recommended, + * [TFLite Model Metadata.](https://www.tensorflow.org/lite/convert/metadata"). + * + * The API supports models with one image input tensor and one or more output tensors. To be more + * specific, here are the requirements. + * + * Input tensor + * (kTfLiteUInt8/kTfLiteFloat32) + * - image input of size `[batch x height x width x channels]`. + * - batch inference is not supported (`batch` is required to be 1). + * - only RGB inputs are supported (`channels` is required to be 3). + * - if type is kTfLiteFloat32, NormalizationOptions are required to be attached to the metadata + * for input normalization. + * + * At least one output tensor with: + * (kTfLiteUInt8/kTfLiteFloat32) + * - `N `classes and either 2 or 4 dimensions, i.e. `[1 x N]` or `[1 x 1 x 1 x N]` + * - optional (but recommended) label map(s) as AssociatedFiles with type TENSOR_AXIS_LABELS, + * containing one label per line. The first such AssociatedFile (if any) is used to fill the + * `class_name` field of the results. The `display_name` field is filled from the AssociatedFile + * (if any) whose locale matches the `display_names_locale` field of the `ImageClassifierOptions` + * used at creation time ("en" by default, i.e. English). If none of these are available, only + * the `index` field of the results will be filled. + * - optional score calibration can be attached using ScoreCalibrationOptions and an AssociatedFile + * with type TENSOR_AXIS_SCORE_CALIBRATION. See metadata_schema.fbs [1] for more details. + */ +NS_SWIFT_NAME(ImageClassifier) +@interface MPPImageClassifier : NSObject + +/** + * Creates a new instance of `MPPImageClassifier` from an absolute path to a TensorFlow Lite + * model file stored locally on the device and the default `MPPImageClassifierOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * image classifier. + * + * @return A new instance of `MPPImageClassifier` with the given model path. `nil` if there is an + * error in initializing the image classifier. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPImageClassifier` from the given `MPPImageClassifierOptions`. + * + * @param options The options of type `MPPImageClassifierOptions` to use for configuring the + * `MPPImageClassifier`. + * @param error An optional error parameter populated when there is an error in initializing the + * image classifier. + * + * @return A new instance of `MPPImageClassifier` with the given options. `nil` if there is an + * error in initializing the image classifier. + */ +- (nullable instancetype)initWithOptions:(MPPImageClassifierOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs image classification on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeImage`. + * + * @param image The `MPPImage` on which image classification is to be performed. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input image. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image + error:(NSError **)error + NS_SWIFT_NAME(classify(image:)); + +/** + * Performs image classification on the provided `MPPImage` cropped to the specified region of + * interest. Rotation will be applied on the cropped image according to the `orientation` property + * of the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeImage`. + * + * @param image The `MPPImage` on which image classification is to be performed. + * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which + * image classification should be performed. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input image. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(classify(image:regionOfInterest:)); + +/** + * Performs image classification on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeImage`. + * + * @param image The `MPPImage` on which image classification is to be performed. + * @param timeStampMs The video frame's timestamp (in milliseconds). The input timestamps must be + * monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input video frame. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error + NS_SWIFT_NAME(classify(videoFrame:timeStampMs:)); + +/** + * Performs image classification on the provided video frame of type `MPPImage` cropped to the + * specified region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeVideo`. + * + * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must + * be monotonically increasing. + * + * @param image A live stream image data of type `MPPImage` on which image classification is to be + * performed. + * @param timeStampMs The video frame's timestamp (in milliseconds). The input timestamps must be + * monotonically increasing. + * @param roi A `CGRect` specifying the region of interest within the video frame of type + * `MPPImage`, on which image classification should be performed. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input video frame. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(classify(videoFrame:timeStampMs:regionOfInterest:)); + +/** + * Sends live stream image data of type `MPPImage` to perform image classification using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeLiveStream`. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the image classifier. The input timestamps must be monotonically increasing. + * + * @param image A live stream image data of type `MPPImage` on which image classification is to be + * performed. + * @param timeStampMs The timestamp (in milliseconds) which indicates when the input image is sent + * to the image classifier. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input live stream image data. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (BOOL)classifyAsyncImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error NS_SWIFT_NAME(classifyAsync(image:timeStampMs:)); + +/** + * Sends live stream image data of type `MPPImage` to perform image classification, cropped to the + * specified region of interest.. Rotation will be applied according to the `orientation` property + * of the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * `MPPRunningModeLiveStream`. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the image classifier. The input timestamps must be monotonically increasing. + * + * @param image A live stream image data of type `MPPImage` on which image classification is to be + * performed. + * @param timeStampMs The timestamp (in milliseconds) which indicates when the input image is sent + * to the image classifier. The input timestamps must be monotonically increasing. + * @param roi A `CGRect` specifying the region of interest within the given live stream image data + * of type `MPPImage`, on which image classification should be performed. + * @param error An optional error parameter populated when there is an error in performing + * image classification on the input live stream image data. + * + * @return A `MPPImageClassifierResult` object that contains a list of image classifications. + */ +- (BOOL)classifyAsyncImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(classifyAsync(image:timeStampMs:regionOfInterest:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm new file mode 100644 index 00000000..f4e13717 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -0,0 +1,228 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +namespace { +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // namespace + +static NSString *const kClassificationsStreamName = @"classifications_out"; +static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kNormRectName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; + +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.image_classifier.ImageClassifierGraph"; + +@interface MPPImageClassifier () { + /** iOS Text Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; +} +@end + +@implementation MPPImageClassifier + +- (instancetype)initWithOptions:(MPPImageClassifierOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ [NSString + stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName] ] + outputStreams:@[ [NSString stringWithFormat:@"%@:%@", kClassificationsTag, + kClassificationsStreamName] ] + taskOptions:options + enableFlowLimiting:NO + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.completion) { + packetsCallback = [=](absl::StatusOr status_or_packets) { + NSError *callbackError = nil; + MPPImageClassifierResult *result; + if ([MPPCommonUtils checkCppError:status_or_packets.status() toError:&callbackError]) { + result = [MPPImageClassifierResult + imageClassifierResultWithClassificationsPacket: + status_or_packets.value()[kClassificationsStreamName.cppString]]; + } + options.completion(result, callbackError); + }; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:std::move(packetsCallback) + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPImageClassifierOptions *options = [[MPPImageClassifierOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageOrientation:image.orientation + roiAllowed:YES + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, + {kNormRectName.cppString, normalizedRectPacket}}; + + std::optional outputPacketMap = [_visionTaskRunner processPacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return + [MPPImageClassifierResult imageClassifierResultWithClassificationsPacket: + outputPacketMap.value()[kClassificationsStreamName.cppString]]; +} + +- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error { + return [self classifyImage:image regionOfInterest:CGRectZero error:error]; +} + +- (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + regionOfInterest:(CGRect)roi + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageOrientation:image.orientation + roiAllowed:YES + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampMs:timestampMs + error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampMs:timestampMs]; + + PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, + {kNormRectName.cppString, normalizedRectPacket}}; + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return + [MPPImageClassifierResult imageClassifierResultWithClassificationsPacket: + outputPacketMap.value()[kClassificationsStreamName.cppString]]; +} + +- (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error { + return [self classifyVideoFrame:image + timestampMs:timestampMs + regionOfInterest:CGRectZero + error:error]; +} + +- (BOOL)classifyAsyncImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + regionOfInterest:(CGRect)roi + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageOrientation:image.orientation + roiAllowed:YES + error:error]; + if (!rect.has_value()) { + return NO; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampMs:timestampMs + error:error]; + if (imagePacket.IsEmpty()) { + return NO; + } + + Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampMs:timestampMs]; + + PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, + {kNormRectName.cppString, normalizedRectPacket}}; + + return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap error:error]; +} + +- (BOOL)classifyAsyncImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + error:(NSError **)error { + return [self classifyAsyncImage:image + timestampMs:timestampMs + regionOfInterest:CGRectZero + error:error]; +} + +@end From fe7bb92859cb6754686986dddc12b333c74d0e3d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:49:42 +0530 Subject: [PATCH 11/25] Added dependency for image format --- mediapipe/tasks/ios/vision/core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 600ef430..bbcf0dca 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -38,6 +38,7 @@ objc_library( deps = [ "//mediapipe/framework:packet", "//mediapipe/framework:timestamp", + "//mediapipe/framework/formats:image", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core/utils:MPPImageUtils", "//mediapipe/framework/formats:rect_cc_proto", From 33a34de03bcae1fa05583c383f2b0cc308a35f57 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 19:50:00 +0530 Subject: [PATCH 12/25] Updated method signature in MPPTaskRunner --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 2 +- mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index b2881ac0..20720e72 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -91,7 +91,7 @@ NS_ASSUME_NONNULL_BEGIN * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the * runner are illegal and will receive errors. */ -- (absl::Status)close; +- (BOOL)closeWithError:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index a5784633..0813760c 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -63,8 +63,9 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; return [MPPCommonUtils checkCppError:sendStatus toError:error]; } -- (absl::Status)close { - return _cppTaskRunner->Close(); +- (BOOL)closeWithError:(NSError **)error { + absl::Status closeStatus = _cppTaskRunner->Close(); + return [MPPCommonUtils checkCppError:closeStatus toError:error]; } @end From 09fa23088a5a16ec09a344fd85aeaf2ace673843 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 2 Mar 2023 20:01:48 +0530 Subject: [PATCH 13/25] Added TODO --- .../vision/image_classifier/sources/MPPImageClassifierOptions.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h index f7e9a629..2e602204 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h @@ -31,6 +31,7 @@ NS_SWIFT_NAME(ImageClassifierOptions) /** * The user-defined result callback for processing live stream data. The result callback should only * be specified when the running mode is set to the live stream mode. + * TODO: Add parameter `MPPImage` in the callback. */ @property(nonatomic, copy) void (^completion)(MPPImageClassifierResult *result, NSError *error); From 4b54f7f45fec949841de904dc1b146b59352bb32 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 10:57:39 +0530 Subject: [PATCH 14/25] Fixed comments in MPPVisionTaskRunner --- .../tasks/ios/vision/core/sources/MPPVisionTaskRunner.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 5212f65f..99787222 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -58,7 +58,7 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * Creates a `NormalizedRect` from region of interest and image orientation, performing + * Creates a `NormalizedRect` from a region of interest and an image orientation, performing * sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns a default * `NormalizedRect` covering the whole image with rotation set according `imageOrientation`. If * `roiAllowed` is NO, an error will be returned if the input region of interest is not equal to @@ -66,11 +66,10 @@ NS_ASSUME_NONNULL_BEGIN * * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals - * `CGRectZero` if `roiAllowed` is NO. Otherwise an error will be returned. + * `CGRectZero` if `roiAllowed` is NO. Otherwise, an error will be returned. * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. * @param roiAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. - * * @param error Pointer to the memory location where errors if any should be * saved. If @c NULL, no error will be saved. * @@ -83,7 +82,7 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error; /** - * A synchronous method to invoke the C++ task runner to process single image inputs. The call + * A synchronous method to invoke the C++ task runner to process single image inputs. The call * blocks the current thread until a failure status or a successful result is returned. * * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. @@ -97,7 +96,7 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error; /** - * A synchronous method to invoke the C++ task runner to process continuous video frames. The call + * A synchronous method to invoke the C++ task runner to process continuous video frames. The call * blocks the current thread until a failure status or a successful result is returned. * * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. From af82dc5e175b4e377c2728e0faf37c950bfbe229 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 11:10:20 +0530 Subject: [PATCH 15/25] Updated comments in MPPImageClassifier --- .../sources/MPPImageClassifier.h | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h index 1914f9ae..0e42ec20 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h @@ -54,8 +54,8 @@ NS_SWIFT_NAME(ImageClassifier) @interface MPPImageClassifier : NSObject /** - * Creates a new instance of `MPPImageClassifier` from an absolute path to a TensorFlow Lite - * model file stored locally on the device and the default `MPPImageClassifierOptions`. + * Creates a new instance of `MPPImageClassifier` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPImageClassifierOptions`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. * @param error An optional error parameter populated when there is an error in initializing the @@ -74,8 +74,8 @@ NS_SWIFT_NAME(ImageClassifier) * @param error An optional error parameter populated when there is an error in initializing the * image classifier. * - * @return A new instance of `MPPImageClassifier` with the given options. `nil` if there is an - * error in initializing the image classifier. + * @return A new instance of `MPPImageClassifier` with the given options. `nil` if there is an error + * in initializing the image classifier. */ - (nullable instancetype)initWithOptions:(MPPImageClassifierOptions *)options error:(NSError **)error NS_DESIGNATED_INITIALIZER; @@ -87,8 +87,8 @@ NS_SWIFT_NAME(ImageClassifier) * `MPPRunningModeImage`. * * @param image The `MPPImage` on which image classification is to be performed. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input image. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input image. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ @@ -105,8 +105,8 @@ NS_SWIFT_NAME(ImageClassifier) * @param image The `MPPImage` on which image classification is to be performed. * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which * image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input image. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input image. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ @@ -119,20 +119,20 @@ NS_SWIFT_NAME(ImageClassifier) * Performs image classification on the provided video frame of type `MPPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeImage`. + * `MPPRunningModeVideo`. * * @param image The `MPPImage` on which image classification is to be performed. - * @param timeStampMs The video frame's timestamp (in milliseconds). The input timestamps must be + * @param timestampMs The video frame's timestamp (in milliseconds). The input timestamps must be * monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input video frame. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input video frame. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image timestampMs:(NSInteger)timestampMs error:(NSError **)error - NS_SWIFT_NAME(classify(videoFrame:timeStampMs:)); + NS_SWIFT_NAME(classify(videoFrame:timestampMs:)); /** * Performs image classification on the provided video frame of type `MPPImage` cropped to the @@ -145,12 +145,12 @@ NS_SWIFT_NAME(ImageClassifier) * * @param image A live stream image data of type `MPPImage` on which image classification is to be * performed. - * @param timeStampMs The video frame's timestamp (in milliseconds). The input timestamps must be + * @param timestampMs The video frame's timestamp (in milliseconds). The input timestamps must be * monotonically increasing. * @param roi A `CGRect` specifying the region of interest within the video frame of type * `MPPImage`, on which image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input video frame. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input video frame. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ @@ -158,7 +158,7 @@ NS_SWIFT_NAME(ImageClassifier) timestampMs:(NSInteger)timestampMs regionOfInterest:(CGRect)roi error:(NSError **)error - NS_SWIFT_NAME(classify(videoFrame:timeStampMs:regionOfInterest:)); + NS_SWIFT_NAME(classify(videoFrame:timestampMs:regionOfInterest:)); /** * Sends live stream image data of type `MPPImage` to perform image classification using the whole @@ -171,16 +171,16 @@ NS_SWIFT_NAME(ImageClassifier) * * @param image A live stream image data of type `MPPImage` on which image classification is to be * performed. - * @param timeStampMs The timestamp (in milliseconds) which indicates when the input image is sent + * @param timestampMs The timestamp (in milliseconds) which indicates when the input image is sent * to the image classifier. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input live stream image data. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input live stream image data. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ - (BOOL)classifyAsyncImage:(MPPImage *)image timestampMs:(NSInteger)timestampMs - error:(NSError **)error NS_SWIFT_NAME(classifyAsync(image:timeStampMs:)); + error:(NSError **)error NS_SWIFT_NAME(classifyAsync(image:timestampMs:)); /** * Sends live stream image data of type `MPPImage` to perform image classification, cropped to the @@ -193,12 +193,12 @@ NS_SWIFT_NAME(ImageClassifier) * * @param image A live stream image data of type `MPPImage` on which image classification is to be * performed. - * @param timeStampMs The timestamp (in milliseconds) which indicates when the input image is sent + * @param timestampMs The timestamp (in milliseconds) which indicates when the input image is sent * to the image classifier. The input timestamps must be monotonically increasing. * @param roi A `CGRect` specifying the region of interest within the given live stream image data * of type `MPPImage`, on which image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing - * image classification on the input live stream image data. + * @param error An optional error parameter populated when there is an error in performing image + * classification on the input live stream image data. * * @return A `MPPImageClassifierResult` object that contains a list of image classifications. */ @@ -206,7 +206,7 @@ NS_SWIFT_NAME(ImageClassifier) timestampMs:(NSInteger)timestampMs regionOfInterest:(CGRect)roi error:(NSError **)error - NS_SWIFT_NAME(classifyAsync(image:timeStampMs:regionOfInterest:)); + NS_SWIFT_NAME(classifyAsync(image:timestampMs:regionOfInterest:)); - (instancetype)init NS_UNAVAILABLE; From d577727698e80c0903dbe350a4c2194241fb6b97 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 11:11:07 +0530 Subject: [PATCH 16/25] Updated MPPImageClassifier --- .../ios/vision/image_classifier/sources/MPPImageClassifier.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index f4e13717..6db7d06c 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -43,7 +43,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.image_classifier.ImageClassifierGraph"; @interface MPPImageClassifier () { - /** iOS Text Task Runner */ + /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; } @end From 8aaabe4a022f1d0c7fae4d13d06c80ab18400b41 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 11:18:36 +0530 Subject: [PATCH 17/25] Updated comments in MPPVisionTaskRunner --- .../ios/vision/core/sources/MPPVisionTaskRunner.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 99787222..007d4f64 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -70,8 +70,8 @@ NS_ASSUME_NONNULL_BEGIN * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. * @param roiAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. - * @param error Pointer to the memory location where errors if any should be - * saved. If @c NULL, no error will be saved. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. * * @return An optional `NormalizedRect` from the given region of interest and image orientation. */ @@ -100,8 +100,8 @@ NS_ASSUME_NONNULL_BEGIN * blocks the current thread until a failure status or a successful result is returned. * * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. - * @param error Pointer to the memory location where errors if any should be - * saved. If @c NULL, no error will be saved. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. * * @return An optional `PacketMap` containing pairs of output stream name and data packet. */ @@ -116,8 +116,8 @@ NS_ASSUME_NONNULL_BEGIN * `MPPVisionTaskRunner`. * * @param packetMap A `PackeMap` containing pairs of input stream name and data packet. - * @param error Pointer to the memory location where errors if any should be - * saved. If @c NULL, no error will be saved. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. * * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. * Please note that any errors during processing of the live stream packet will only be available in From 289b3b20de9a66d89e58e95d132ad0115360bcf8 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 12:01:45 +0530 Subject: [PATCH 18/25] Added methods for common functionality in MPPImageClassifier --- .../sources/MPPImageClassifier.mm | 88 ++++++++++--------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 6db7d06c..564aede8 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -42,6 +42,11 @@ static NSString *const kNormRectTag = @"NORM_RECT"; static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.image_classifier.ImageClassifierGraph"; +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { kNormRectName.cppString, normalizedRectPacket } \ + } + @interface MPPImageClassifier () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -123,8 +128,7 @@ static NSString *const kTaskGraphName = Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, - {kNormRectName.cppString, normalizedRectPacket}}; + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); std::optional outputPacketMap = [_visionTaskRunner processPacketMap:inputPacketMap error:error]; @@ -137,6 +141,33 @@ static NSString *const kTaskGraphName = outputPacketMap.value()[kClassificationsStreamName.cppString]]; } +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampMs:(NSInteger)timestampMs + regionOfInterest:(CGRect)roi + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:roi + imageOrientation:image.orientation + roiAllowed:YES + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampMs:timestampMs + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampMs:timestampMs]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + - (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error { return [self classifyImage:image regionOfInterest:CGRectZero error:error]; } @@ -145,30 +176,17 @@ static NSString *const kTaskGraphName = timestampMs:(NSInteger)timestampMs regionOfInterest:(CGRect)roi error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi - imageOrientation:image.orientation - roiAllowed:YES - error:error]; - if (!rect.has_value()) { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampMs:timestampMs + regionOfInterest:roi + error:error]; + if (!inputPacketMap.has_value()) { return nil; } - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampMs:timestampMs - error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampMs:timestampMs]; - - PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, - {kNormRectName.cppString, normalizedRectPacket}}; - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap error:error]; + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + if (!outputPacketMap.has_value()) { return nil; } @@ -191,29 +209,15 @@ static NSString *const kTaskGraphName = timestampMs:(NSInteger)timestampMs regionOfInterest:(CGRect)roi error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectFromRegionOfInterest:roi - imageOrientation:image.orientation - roiAllowed:YES - error:error]; - if (!rect.has_value()) { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampMs:timestampMs + regionOfInterest:roi + error:error]; + if (!inputPacketMap.has_value()) { return NO; } - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampMs:timestampMs - error:error]; - if (imagePacket.IsEmpty()) { - return NO; - } - - Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampMs:timestampMs]; - - PacketMap inputPacketMap = {{kImageInStreamName.cppString, imagePacket}, - {kNormRectName.cppString, normalizedRectPacket}}; - - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap error:error]; + return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; } - (BOOL)classifyAsyncImage:(MPPImage *)image From 6253966901930f2bf45912330ae8098a34e1a733 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 12:15:25 +0530 Subject: [PATCH 19/25] Updated comments in MPPTaskRunner to include note about mirrored orientations --- .../ios/vision/core/sources/MPPVisionTaskRunner.h | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 007d4f64..ced188e3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -59,16 +59,23 @@ NS_ASSUME_NONNULL_BEGIN /** * Creates a `NormalizedRect` from a region of interest and an image orientation, performing - * sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns a default - * `NormalizedRect` covering the whole image with rotation set according `imageOrientation`. If - * `roiAllowed` is NO, an error will be returned if the input region of interest is not equal to + * sanity checks on-the-fly. + * If the input region of interest equals `CGRectZero`, returns a default `NormalizedRect` covering + * the whole image with rotation set according `imageOrientation`. + * If `roiAllowed` is NO, an error will be returned if the input region of interest is not equal to * `CGRectZero`. + * Mirrored orientations (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`, + * `UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`) are not supported. An error + * will be returned if `imageOrientation` is equal to any one of them. * * @param roi A `CGRect` specifying the region of interest. If the input region of interest equals * `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals * `CGRectZero` if `roiAllowed` is NO. Otherwise, an error will be returned. * @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the * image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise. + * Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`, + * `UIImageOrientationLeftMirrored`, `UIImageOrientationRightMirrored`) are not supported. An error + * will be returned if `imageOrientation` is equal to any one of them. * @param roiAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`. * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no * error will be saved. From fee66069acad274efb98a64da939e0d1264925b1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 12:19:14 +0530 Subject: [PATCH 20/25] Updated error with info about unsupported mirrored orientations in MPPVisionTaskRunner --- .../ios/vision/core/sources/MPPVisionTaskRunner.mm | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 3c115f1c..fff14a97 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -113,9 +113,13 @@ using ::mediapipe::tasks::core::PacketsCallback; break; } default: - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"Unsupported UIImageOrientation."]; + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unsupported UIImageOrientation. `imageOrientation` cannot be equal to " + @"any of the mirrored orientations " + @"(`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,`" + @"UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`)"]; } normalizedRect.set_rotation(-rotationDegrees * M_PI / 180.0); From 160fa424b536f0d6648f0839facd950df3462682 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 12:33:17 +0530 Subject: [PATCH 21/25] Updated documentation of MPPTaskRunner --- .../tasks/ios/core/sources/MPPTaskRunner.h | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 20720e72..df629298 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -64,15 +64,23 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * A synchronous method for processing batch data or offline streaming data. This method is designed - * for processing either batch data such as unrelated images and texts or offline streaming data - * such as the decoded frames from a video file or audio file. The call blocks the current - * thread until a failure status or a successful result is returned. If the input packets have no - * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp - * is set in the input packets, the caller must ensure that the input packet timestamps are greater - * than the timestamps of the previous invocation. This method is thread-unsafe and it is the - * caller's responsibility to synchronize access to this method across multiple threads and to - * ensure that the input packet timestamps are in order. + * A synchronous method for invoking the C++ task runner for processing batch data or offline + * streaming data. This method is designed for processing either batch data such as unrelated images + * and texts or offline streaming data such as the decoded frames from a video file or audio file. + * The call blocks the current thread until a failure status or a successful result is returned. If + * the input packets have no timestamp, an internal timestamp will be assigend per invocation. + * Otherwise, when the timestamp is set in the input packets, the caller must ensure that the input + * packet timestamps are greater than the timestamps of the previous invocation. This method is + * thread-unsafe and it is the caller's responsibility to synchronize access to this method across + * multiple threads and to ensure that the input packet timestamps are in order. + * + * @param packetMap A `PacketMap` containing pairs of input stream name and data packet which are to + * be sent to the C++ task runner for processing synchronously. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An optional output `PacketMap` containing pairs of output stream name and data packet + * which holds the results of processing the input packet map, if there are no errors. */ - (std::optional) processPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap @@ -84,12 +92,27 @@ NS_ASSUME_NONNULL_BEGIN * packets. The caller must ensure that the input packet timestamps are monotonically increasing. * This method is thread-unsafe and it is the caller's responsibility to synchronize access to this * method across multiple threads and to ensure that the input packet timestamps are in order. + * + * @param packetMap A `PacketMap` containing pairs of input stream name and data packet that are to + * be sent to the C++ task runner for processing asynchronously. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. + * Please note that any errors during processing of the live stream packet map will only be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. */ - (BOOL)sendPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap error:(NSError **)error; /** * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the * runner are illegal and will receive errors. + * + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the C++ task runner was shutdown successfully. */ - (BOOL)closeWithError:(NSError **)error; From f2dfa7f474d467b57719fe01b99539c4ff2fa4ae Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 12:33:31 +0530 Subject: [PATCH 22/25] Updated documentation of MPPVisionTaskRunner --- mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index ced188e3..6e4078f3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -127,7 +127,7 @@ NS_ASSUME_NONNULL_BEGIN * error will be saved. * * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. - * Please note that any errors during processing of the live stream packet will only be available in + * Please note that any errors during processing of the live stream packet map will only be available in * the user-defined `packetsCallback` that was provided during initialization of the * `MPPVisionTaskRunner`. */ From 412476eba11d292b0380c88a22b8a93b02f2cc3e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 14:39:03 +0530 Subject: [PATCH 23/25] Added inline function to return display name of MPPRunningMode --- .../ios/vision/core/sources/MPPRunningMode.h | 19 +++++++++++++++ .../core/sources/MPPVisionTaskRunner.mm | 24 +++++++++++-------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h index 5cc57b88..8818d9b1 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -38,4 +38,23 @@ typedef NS_ENUM(NSUInteger, MPPRunningMode) { } NS_SWIFT_NAME(RunningMode); +NS_INLINE NSString *MPPRunningModeDisplayName(MPPRunningMode runningMode) { + + if (runningMode > MPPRunningModeLiveStream) { + return nil; + } + + #define MPPRunningModeDisplayNameMap(mode) [mode] = @#mode + + NSString *displayNameMap[MPPRunningModeLiveStream + 1] = { + MPPRunningModeDisplayNameMap(MPPRunningModeImage), + MPPRunningModeDisplayNameMap(MPPRunningModeVideo), + MPPRunningModeDisplayNameMap(MPPRunningModeLiveStream), + }; + + #undef MPPRunningModeDisplayNameMap + + return displayNameMap[runningMode]; +} + NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index fff14a97..43ef8ce9 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -116,10 +116,11 @@ using ::mediapipe::tasks::core::PacketsCallback; [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"Unsupported UIImageOrientation. `imageOrientation` cannot be equal to " - @"any of the mirrored orientations " - @"(`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,`" - @"UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`)"]; + description: + @"Unsupported UIImageOrientation. `imageOrientation` cannot be equal to " + @"any of the mirrored orientations " + @"(`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,`" + @"UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`)"]; } normalizedRect.set_rotation(-rotationDegrees * M_PI / 180.0); @@ -132,8 +133,9 @@ using ::mediapipe::tasks::core::PacketsCallback; [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"The vision task is not initialized with image mode. Current Running Mode:"]; + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"image mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; return std::nullopt; } @@ -146,8 +148,9 @@ using ::mediapipe::tasks::core::PacketsCallback; [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"The vision task is not initialized with image mode. Current Running Mode:"]; + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"video mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; return std::nullopt; } @@ -159,8 +162,9 @@ using ::mediapipe::tasks::core::PacketsCallback; [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"The vision task is not initialized with image mode. Current Running Mode:"]; + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"live stream mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; return NO; } From 7b8d92ba47199921a47f99e565c8b2c17d3bc63d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 14:39:22 +0530 Subject: [PATCH 24/25] Updated formatting in MPPVisionTaskRunner --- mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 6e4078f3..52a9e6d3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -127,8 +127,8 @@ NS_ASSUME_NONNULL_BEGIN * error will be saved. * * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. - * Please note that any errors during processing of the live stream packet map will only be available in - * the user-defined `packetsCallback` that was provided during initialization of the + * Please note that any errors during processing of the live stream packet map will only be + * available in the user-defined `packetsCallback` that was provided during initialization of the * `MPPVisionTaskRunner`. */ - (BOOL)processLiveStreamPacketMap:(const mediapipe::tasks::core::PacketMap &)packetMap From 87df23f0fa51072ee1cd9e757bb72ba11704c5f4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 3 Mar 2023 14:40:00 +0530 Subject: [PATCH 25/25] Updated formatting of MPPRunningMode.h --- mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h index 8818d9b1..4abacc1a 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -39,20 +39,19 @@ typedef NS_ENUM(NSUInteger, MPPRunningMode) { } NS_SWIFT_NAME(RunningMode); NS_INLINE NSString *MPPRunningModeDisplayName(MPPRunningMode runningMode) { - if (runningMode > MPPRunningModeLiveStream) { - return nil; + return nil; } - #define MPPRunningModeDisplayNameMap(mode) [mode] = @#mode - +#define MPPRunningModeDisplayNameMap(mode) [mode] = @ #mode + NSString *displayNameMap[MPPRunningModeLiveStream + 1] = { MPPRunningModeDisplayNameMap(MPPRunningModeImage), MPPRunningModeDisplayNameMap(MPPRunningModeVideo), MPPRunningModeDisplayNameMap(MPPRunningModeLiveStream), }; - #undef MPPRunningModeDisplayNameMap +#undef MPPRunningModeDisplayNameMap return displayNameMap[runningMode]; }