From 47e7ec47a28b826c46b200515ac815d48e85f2c4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 14:04:09 +0530 Subject: [PATCH 1/8] Changed delegate method to optional --- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h index f1ba5411..71f65253 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -33,7 +33,7 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) @protocol MPPImageSegmenterLiveStreamDelegate -@required +@optional /** * This method notifies a delegate that the results of asynchronous segmentation of From f74f7b86570ba6aec89584165e9e0dba3a91f2a6 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 14:04:17 +0530 Subject: [PATCH 2/8] Fixed typo --- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m index 282a729b..8dd58357 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m @@ -30,7 +30,7 @@ imageSegmenterOptions.runningMode = self.runningMode; imageSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; - imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputConfidenceMasks; + imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputCategoryMasks; imageSegmenterOptions.displayNamesLocale = self.displayNamesLocale; imageSegmenterOptions.imageSegmenterLiveStreamDelegate = self.imageSegmenterLiveStreamDelegate; From 0863d8def58ae4fc9a736e51e781ab38a163039f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 14:04:34 +0530 Subject: [PATCH 3/8] Added iOS image segmenter implementation file --- .../sources/MPPImageSegmenter.mm | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm new file mode 100644 index 00000000..a17001fe --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -0,0 +1,261 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h" + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +// Constants for the underlying MP Tasks Graph. See +// https://github.com/google/mediapipe/tree/master/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc +static NSString *const kConfidenceMasksStreamName = @"confidence_masks"; +static NSString *const kConfidenceMasksTag = @"CONFIDENCE_MASKS"; +static NSString *const kCategoryMaskStreamName = @"category_mask"; +static NSString *const kCategoryMaskTag = @"CATEGORY_MASK"; +static NSString *const kQualityScoresStreamName = @"quality_scores"; +static NSString *const kQualityScoresTag = @"QUALITY_SCORES"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.image_segmenter.ImageSegmenterGraph"; +static NSString *const kTaskName = @"imageSegmenter"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectStreamName.cppString, normalizedRectPacket \ + } \ + } + +namespace { +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // anonymous namespace + +@interface MPPImageSegmenter () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; +} +@property(nonatomic, weak) id imageSegmenterLiveStreamDelegate; + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult; +@end + +@implementation MPPImageSegmenter + +- (nullable MPPImageSegmenterResult *) + imageSegmenterResultWithOutputPacketMap:(PacketMap &)outputPacketMap + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { + return [MPPImageSegmenterResult + imageSegmenterResultWithConfidenceMasksPacket:outputPacketMap[kConfidenceMasksStreamName + .cppString] + categoryMaskPacket:outputPacketMap[kCategoryMaskStreamName + .cppString] + qualityScoresPacket:outputPacketMap[kQualityScoresStreamName + .cppString] + timestampInMilliseconds:outputPacketMap[kImageOutStreamName.cppString] + .Timestamp() + .Value() / + kMicrosecondsPerMillisecond + shouldCopyMaskPacketData:shouldCopyMaskPacketData]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.imageSegmenterLiveStreamDelegate + respondsToSelector:@selector(imageSegmenter: + didFinishSegmentationWithResult:timestampInMilliseconds:error:)]) { + return; + } + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.imageSegmenterLiveStreamDelegate imageSegmenter:self + didFinishSegmentationWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPImageSegmenterResult *result = [self imageSegmenterResultWithOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; + + dispatch_async(_callbackQueue, ^{ + [self.imageSegmenterLiveStreamDelegate imageSegmenter:self + didFinishSegmentationWithResult:result + timestampInMilliseconds:result.timestampInMilliseconds + error:callbackError]; + }); +} + +- (instancetype)initWithOptions:(MPPImageSegmenterOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + NSMutableArray *outputStreams = [NSMutableArray + arrayWithObjects:[NSString + stringWithFormat:@"%@:%@", kQualityScoresTag, kQualityScoresStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName], + nil]; + if (options.shouldOutputConfidenceMasks) { + [outputStreams addObject:[NSString + stringWithFormat:@"%@:%@", kConfidenceMasksTag, kConfidenceMasksStreamName]]; + } + if (options.shouldOutputCategoryMask) { + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kCategoryMaskTag, + kCategoryMaskStreamName]]; + } + + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] + ] + outputStreams:outputStreams + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.imageSegmenterLiveStreamDelegate) { + _imageSegmenterLiveStreamDelegate = options.imageSegmenterLiveStreamDelegate; + + // Create a private serial dispatch queue in which the delegate method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPImageSegmenter *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPImageSegmenterOptions *options = [[MPPImageSegmenterOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPImageSegmenterResult *) + imageSegmenterResultWithOptionalOutputPacketMap:(std::optional &)outputPacketMap + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { + if (!outputPacketMap.has_value()) { + return nil; + } + MPPImageSegmenterResult *result = + [self imageSegmenterResultWithOutputPacketMap:outputPacketMap.value() + shouldCopyMaskPacketData:shouldCopyMaskPacketData]; + return result; +} + +- (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image error:(NSError **)error { + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; + return [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:YES]; +} + +- (void)segmentImage:(MPPImage *)image + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler { + NSError *error = nil; + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:&error]; + + MPPImageSegmenterResult *result = + [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; + completionHandler(result, error); +} + +- (nullable MPPImageSegmenterResult *)segmentVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional outputPacketMap = + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + + return [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:YES]; +} + +- (void)segmentVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler { + NSError *error = nil; + std::optional outputPacketMap = + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:&error]; + + MPPImageSegmenterResult *result = + [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; + completionHandler(result, error); +} + +- (BOOL)segmentAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +@end From 5a1564e04c0149aa5e5c6de784a91d6c0716efd7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 14:22:17 +0530 Subject: [PATCH 4/8] Updated image segmenter bazel target to add MPPImageSegmenter.mm --- mediapipe/tasks/ios/vision/image_segmenter/BUILD | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/BUILD index 54031f24..ce05052b 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/ios/vision/image_segmenter/BUILD @@ -38,10 +38,24 @@ objc_library( objc_library( name = "MPPImageSegmenter", - hdrs = ["sources/MPPImageSegmenterOptions.h"], + hdrs = ["sources/MPPImageSegmenter.h"], + srcs = ["sources/MPPImageSegmenter.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + module_name = "MPPImageSegmenter", deps = [ ":MPPImageSegmenterOptions", ":MPPImageSegmenterResult", "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", + "//mediapipe/tasks/ios/vision/image_segmenter/utils:MPPImageSegmenterOptionsHelpers", + "//mediapipe/tasks/ios/vision/image_segmenter/utils:MPPImageSegmenterResultHelpers", ], ) From bac3efdf6a229c5222b70262095f6fb8d880f385 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 17:31:17 +0530 Subject: [PATCH 5/8] Fixed typo in MPPImageSegmenter.h --- .../ios/vision/image_segmenter/sources/MPPImageSegmenter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h index 819b2012..19e0d6ba 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h @@ -103,7 +103,7 @@ NS_SWIFT_NAME(ImageSegmenter) * The lifetime of the returned masks is only guaranteed for the duration of the block. */ - (void)segmentImage:(MPPImage *)image - withCompletionHandler:((void ^)(MPPImageSegmenterResult *_Nullable result, + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, NSError *_Nullable error))completionHandler NS_SWIFT_NAME(segment(image:completion:)); @@ -163,7 +163,7 @@ NS_SWIFT_NAME(ImageSegmenter) */ - (void)segmentVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds - withCompletionHandler:((void ^)(MPPImageSegmenterResult *_Nullable result, + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, NSError *_Nullable error))completionHandler NS_SWIFT_NAME(segment(videoFrame:timestampInMilliseconds:completion:)); From ec87f068c1b3c105ea1960ed41742b7cbebd0166 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 17:31:54 +0530 Subject: [PATCH 6/8] Renamed option in MPPImageSegmenterOptions --- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.h | 2 +- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.m | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h index 71f65253..b089ac7d 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -92,7 +92,7 @@ NS_SWIFT_NAME(ImageSegmenterOptions) @property(nonatomic) BOOL shouldOutputConfidenceMasks; /** Represents whether to output category mask. */ -@property(nonatomic) BOOL shouldOutputCategoryMasks; +@property(nonatomic) BOOL shouldOutputCategoryMask; @end diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m index 8dd58357..609d64e2 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m @@ -30,7 +30,7 @@ imageSegmenterOptions.runningMode = self.runningMode; imageSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; - imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputCategoryMasks; + imageSegmenterOptions.shouldOutputCategoryMask = self.shouldOutputCategoryMask; imageSegmenterOptions.displayNamesLocale = self.displayNamesLocale; imageSegmenterOptions.imageSegmenterLiveStreamDelegate = self.imageSegmenterLiveStreamDelegate; From 9f015401912d418be0d65bd2bd4505a8ec0c452a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 18:02:34 +0530 Subject: [PATCH 7/8] Changed order of methods in MPPImageSegmenter.mm --- .../sources/MPPImageSegmenter.mm | 155 +++++++++--------- 1 file changed, 79 insertions(+), 76 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index a17001fe..fc17a46c 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -52,7 +52,7 @@ using ::mediapipe::Packet; using ::mediapipe::Timestamp; using ::mediapipe::tasks::core::PacketMap; using ::mediapipe::tasks::core::PacketsCallback; -} // anonymous namespace +} // anonymous namespace @interface MPPImageSegmenter () { /** iOS Vision Task Runner */ @@ -66,67 +66,18 @@ using ::mediapipe::tasks::core::PacketsCallback; @implementation MPPImageSegmenter -- (nullable MPPImageSegmenterResult *) - imageSegmenterResultWithOutputPacketMap:(PacketMap &)outputPacketMap - shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { - return [MPPImageSegmenterResult - imageSegmenterResultWithConfidenceMasksPacket:outputPacketMap[kConfidenceMasksStreamName - .cppString] - categoryMaskPacket:outputPacketMap[kCategoryMaskStreamName - .cppString] - qualityScoresPacket:outputPacketMap[kQualityScoresStreamName - .cppString] - timestampInMilliseconds:outputPacketMap[kImageOutStreamName.cppString] - .Timestamp() - .Value() / - kMicrosecondsPerMillisecond - shouldCopyMaskPacketData:shouldCopyMaskPacketData]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.imageSegmenterLiveStreamDelegate - respondsToSelector:@selector(imageSegmenter: - didFinishSegmentationWithResult:timestampInMilliseconds:error:)]) { - return; - } - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.imageSegmenterLiveStreamDelegate imageSegmenter:self - didFinishSegmentationWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPImageSegmenterResult *result = [self imageSegmenterResultWithOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:NO]; - - dispatch_async(_callbackQueue, ^{ - [self.imageSegmenterLiveStreamDelegate imageSegmenter:self - didFinishSegmentationWithResult:result - timestampInMilliseconds:result.timestampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPImageSegmenterOptions *)options error:(NSError **)error { self = [super init]; if (self) { NSMutableArray *outputStreams = [NSMutableArray - arrayWithObjects:[NSString - stringWithFormat:@"%@:%@", kQualityScoresTag, kQualityScoresStreamName], - [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName], - nil]; + arrayWithObjects:[NSString stringWithFormat:@"%@:%@", kQualityScoresTag, + kQualityScoresStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName], nil]; if (options.shouldOutputConfidenceMasks) { - [outputStreams addObject:[NSString - stringWithFormat:@"%@:%@", kConfidenceMasksTag, kConfidenceMasksStreamName]]; + [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kConfidenceMasksTag, + kConfidenceMasksStreamName]]; } if (options.shouldOutputCategoryMask) { [outputStreams addObject:[NSString stringWithFormat:@"%@:%@", kCategoryMaskTag, @@ -192,22 +143,10 @@ using ::mediapipe::tasks::core::PacketsCallback; return [self initWithOptions:options error:error]; } -- (nullable MPPImageSegmenterResult *) - imageSegmenterResultWithOptionalOutputPacketMap:(std::optional &)outputPacketMap - shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { - if (!outputPacketMap.has_value()) { - return nil; - } - MPPImageSegmenterResult *result = - [self imageSegmenterResultWithOutputPacketMap:outputPacketMap.value() - shouldCopyMaskPacketData:shouldCopyMaskPacketData]; - return result; -} - - (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - return [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:YES]; + return [MPPImageSegmenter imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:YES]; } - (void)segmentImage:(MPPImage *)image @@ -217,8 +156,8 @@ using ::mediapipe::tasks::core::PacketsCallback; std::optional outputPacketMap = [_visionTaskRunner processImage:image error:&error]; MPPImageSegmenterResult *result = - [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:NO]; + [MPPImageSegmenter imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; completionHandler(result, error); } @@ -230,8 +169,8 @@ using ::mediapipe::tasks::core::PacketsCallback; timestampInMilliseconds:timestampInMilliseconds error:error]; - return [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:YES]; + return [MPPImageSegmenter imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:YES]; } - (void)segmentVideoFrame:(MPPImage *)image @@ -245,8 +184,8 @@ using ::mediapipe::tasks::core::PacketsCallback; error:&error]; MPPImageSegmenterResult *result = - [self imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:NO]; + [MPPImageSegmenter imageSegmenterResultWithOptionalOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; completionHandler(result, error); } @@ -258,4 +197,68 @@ using ::mediapipe::tasks::core::PacketsCallback; error:error]; } +#pragma mark - Private + ++ (nullable MPPImageSegmenterResult *) + imageSegmenterResultWithOptionalOutputPacketMap:(std::optional &)outputPacketMap + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { + if (!outputPacketMap.has_value()) { + return nil; + } + MPPImageSegmenterResult *result = + [self imageSegmenterResultWithOutputPacketMap:outputPacketMap.value() + shouldCopyMaskPacketData:shouldCopyMaskPacketData]; + return result; +} + ++ (nullable MPPImageSegmenterResult *) + imageSegmenterResultWithOutputPacketMap:(PacketMap &)outputPacketMap + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { + return [MPPImageSegmenterResult + imageSegmenterResultWithConfidenceMasksPacket:outputPacketMap[kConfidenceMasksStreamName + .cppString] + categoryMaskPacket:outputPacketMap[kCategoryMaskStreamName + .cppString] + qualityScoresPacket:outputPacketMap[kQualityScoresStreamName + .cppString] + timestampInMilliseconds:outputPacketMap[kImageOutStreamName.cppString] + .Timestamp() + .Value() / + kMicrosecondsPerMillisecond + shouldCopyMaskPacketData:shouldCopyMaskPacketData]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.imageSegmenterLiveStreamDelegate + respondsToSelector:@selector(imageSegmenter: + didFinishSegmentationWithResult:timestampInMilliseconds:error:)]) { + return; + } + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.imageSegmenterLiveStreamDelegate imageSegmenter:self + didFinishSegmentationWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPImageSegmenterResult *result = [self imageSegmenterResultWithOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; + + dispatch_async(_callbackQueue, ^{ + [self.imageSegmenterLiveStreamDelegate imageSegmenter:self + didFinishSegmentationWithResult:result + timestampInMilliseconds:result.timestampInMilliseconds + error:callbackError]; + }); +} + @end From d16cb724383f591bed422ae321e2287ffb4f4259 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 31 Aug 2023 18:04:09 +0530 Subject: [PATCH 8/8] Fixed method call in MPPImageSegmenter.mm --- .../ios/vision/image_segmenter/sources/MPPImageSegmenter.mm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index fc17a46c..be7c8beb 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -250,8 +250,9 @@ using ::mediapipe::tasks::core::PacketsCallback; return; } - MPPImageSegmenterResult *result = [self imageSegmenterResultWithOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:NO]; + MPPImageSegmenterResult *result = + [MPPImageSegmenter imageSegmenterResultWithOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; dispatch_async(_callbackQueue, ^{ [self.imageSegmenterLiveStreamDelegate imageSegmenter:self