From f63c00b3c62a95fa47d5bfcfc5104c373fd0605b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 8 Jun 2023 18:09:42 +0530 Subject: [PATCH 1/2] Added hand landmarker implementation file and hand landmarker connections --- .../tasks/ios/vision/hand_landmarker/BUILD | 16 + .../sources/MPPHandLandmarker.mm | 288 ++++++++++++++++++ .../sources/MPPHandLandmarksConnections.h | 58 ++++ 3 files changed, 362 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm create mode 100644 mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD index 6c815f28..d081ab96 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD @@ -38,8 +38,15 @@ objc_library( ], ) +objc_library( + name = "MPPHandLandmarksConnections", + hdrs = ["sources/MPPHandLandmarksConnections.h"], + module_name = "MPPHandLandmarksConnections", +) + objc_library( name = "MPPHandLandmarker", + srcs = ["sources/MPPHandLandmarker.mm"], hdrs = ["sources/MPPHandLandmarker.h"], copts = [ "-ObjC++", @@ -48,10 +55,19 @@ objc_library( ], module_name = "MPPHandLandmarker", deps = [ + ":MPPHandLandmarksConnections", ":MPPHandLandmarkerOptions", ":MPPHandLandmarkerResult", "//mediapipe/tasks/ios/components/containers:MPPConnection", "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarker_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerOptionsHelpers", + "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerResultHelpers", ], ) diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm new file mode 100644 index 00000000..143b9bd2 --- /dev/null +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm @@ -0,0 +1,288 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+Helpers.h" + +namespace { +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // namespace + +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kNormRectInStreamName = @"norm_rect_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kLandmarksTag = @"LANDMARKS"; +static NSString *const kLandmarksOutStreamName = @"hand_landmarks"; +static NSString *const kWorldLandmarksTag = @"WORLD_LANDMARKS"; +static NSString *const kWorldLandmarksOutStreamName = @"world_hand_landmarks"; +static NSString *const kHandednessTag = @"HANDEDNESS"; +static NSString *const kHandednessOutStreamName = @"handedness"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.hand_landmarker.HandLandmarkerGraph"; +static NSString *const kTaskName = @"handLandmarker"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectInStreamName.cppString, normalizedRectPacket \ + } \ + } + +@interface MPPHandLandmarker () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; +} +@property(nonatomic, weak) id handLandmarkerLiveStreamDelegate; +@end + +@implementation MPPHandLandmarker + +- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOutputPacketMap: + (PacketMap &)outputPacketMap { + return [MPPHandLandmarkerResult + handLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName + .cppString] + handednessPacket:outputPacketMap[kHandednessOutStreamName.cppString]]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.handLandmarkerLiveStreamDelegate + respondsToSelector:@selector(handLandmarker: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPHandLandmarkerResult *result = [self handLandmarkerResultWithOutputPacketMap:outputPacketMap]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + +- (instancetype)initWithOptions:(MPPHandLandmarkerOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectInStreamName] + ] + outputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kLandmarksTag, kLandmarksOutStreamName], + [NSString + stringWithFormat:@"%@:%@", kWorldLandmarksTag, kWorldLandmarksOutStreamName], + [NSString stringWithFormat:@"%@:%@", kHandednessTag, kHandednessOutStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName] + ] + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.handLandmarkerLiveStreamDelegate) { + _handLandmarkerLiveStreamDelegate = options.handLandmarkerLiveStreamDelegate; + + // Create a private serial dispatch queue in which the deleagte method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPHandLandmarker *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:std::move(packetsCallback) + error:error]; + if (!_visionTaskRunner) { + return nil; + } + } + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPHandLandmarkerOptions *options = [[MPPHandLandmarkerOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + MPPHandLandmarkerResult *result = + [self handLandmarkerResultWithOutputPacketMap:outputPacketMap.value()]; + return result; +} + +- (nullable MPPHandLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation + imageSize:CGSizeMake(image.width, image.height) + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (nullable MPPHandLandmarkerResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return nil; + } + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + + return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; +} + +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return NO; + } + + return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; +} + ++ (NSArray *)handPalmConnections { + return kHandPalmConnections; +} + ++ (NSArray *)handIndexFingerConnections { + return kHandIndexFingerConnections; +} + ++ (NSArray *)handMiddleFingerConnections { + return kHandMiddleFingerConnections; +} + ++ (NSArray *)handRingFingerConnections { + return kHandRingFingerConnections; +} + ++ (NSArray *)handPinkyConnections { + return kHandPinkyConnections; +} + ++ (NSArray *)handConnections { + return kHandConnections; +} + +@end diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h new file mode 100644 index 00000000..fe356fbe --- /dev/null +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h @@ -0,0 +1,58 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPConnection.h" + +NS_ASSUME_NONNULL_BEGIN + +NSArray *const kHandPalmConnections = @[ + [[MPPConnection alloc] initWithStart:0 end:1], [[MPPConnection alloc] initWithStart:0 end:5], + [[MPPConnection alloc] initWithStart:9 end:13], [[MPPConnection alloc] initWithStart:13 end:17], + [[MPPConnection alloc] initWithStart:5 end:9], [[MPPConnection alloc] initWithStart:0 end:17] +]; + +NSArray *const kHandThumbConnections = @[ + [[MPPConnection alloc] initWithStart:1 end:2], [[MPPConnection alloc] initWithStart:2 end:3], + [[MPPConnection alloc] initWithStart:3 end:4] +]; + +NSArray *const kHandIndexFingerConnections = @[ + [[MPPConnection alloc] initWithStart:5 end:6], [[MPPConnection alloc] initWithStart:6 end:7], + [[MPPConnection alloc] initWithStart:7 end:8] +]; + +NSArray *const kHandMiddleFingerConnections = @[ + [[MPPConnection alloc] initWithStart:9 end:10], [[MPPConnection alloc] initWithStart:10 end:11], + [[MPPConnection alloc] initWithStart:11 end:12] +]; + +NSArray *const kHandRingFingerConnections = @[ + [[MPPConnection alloc] initWithStart:13 end:14], [[MPPConnection alloc] initWithStart:14 end:15], + [[MPPConnection alloc] initWithStart:15 end:16] +]; + +NSArray *const kHandPinkyConnections = @[ + [[MPPConnection alloc] initWithStart:16 end:17], [[MPPConnection alloc] initWithStart:17 end:18], + [[MPPConnection alloc] initWithStart:18 end:19] +]; + +NSArray *const kHandConnections = [[[[[[NSArray + arrayWithArray:kHandPalmConnections] arrayByAddingObjectsFromArray:kHandThumbConnections] + arrayByAddingObjectsFromArray:kHandIndexFingerConnections] + arrayByAddingObjectsFromArray:kHandMiddleFingerConnections] + arrayByAddingObjectsFromArray:kHandRingFingerConnections] + arrayByAddingObjectsFromArray:kHandPinkyConnections]; + +NS_ASSUME_NONNULL_END From f528fa5de20a6eb8b43e7d2bd8d668ebf23b456f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 9 Jun 2023 17:30:23 +0530 Subject: [PATCH 2/2] Updated constant names in MPPHandLandmarkConnections --- .../hand_landmarker/sources/MPPHandLandmarker.mm | 12 ++++++------ .../sources/MPPHandLandmarksConnections.h | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm index 143b9bd2..46a6d1f3 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm @@ -262,27 +262,27 @@ static NSString *const kTaskName = @"handLandmarker"; } + (NSArray *)handPalmConnections { - return kHandPalmConnections; + return MPPHandPalmConnections; } + (NSArray *)handIndexFingerConnections { - return kHandIndexFingerConnections; + return MPPHandIndexFingerConnections; } + (NSArray *)handMiddleFingerConnections { - return kHandMiddleFingerConnections; + return MPPHandMiddleFingerConnections; } + (NSArray *)handRingFingerConnections { - return kHandRingFingerConnections; + return MPPHandRingFingerConnections; } + (NSArray *)handPinkyConnections { - return kHandPinkyConnections; + return MPPHandPinkyConnections; } + (NSArray *)handConnections { - return kHandConnections; + return MPPHandConnections; } @end diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h index fe356fbe..7027c4ff 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h @@ -17,38 +17,38 @@ NS_ASSUME_NONNULL_BEGIN -NSArray *const kHandPalmConnections = @[ +NSArray *const MPPHandPalmConnections = @[ [[MPPConnection alloc] initWithStart:0 end:1], [[MPPConnection alloc] initWithStart:0 end:5], [[MPPConnection alloc] initWithStart:9 end:13], [[MPPConnection alloc] initWithStart:13 end:17], [[MPPConnection alloc] initWithStart:5 end:9], [[MPPConnection alloc] initWithStart:0 end:17] ]; -NSArray *const kHandThumbConnections = @[ +NSArray *const MPPHandThumbConnections = @[ [[MPPConnection alloc] initWithStart:1 end:2], [[MPPConnection alloc] initWithStart:2 end:3], [[MPPConnection alloc] initWithStart:3 end:4] ]; -NSArray *const kHandIndexFingerConnections = @[ +NSArray *const MPPHandIndexFingerConnections = @[ [[MPPConnection alloc] initWithStart:5 end:6], [[MPPConnection alloc] initWithStart:6 end:7], [[MPPConnection alloc] initWithStart:7 end:8] ]; -NSArray *const kHandMiddleFingerConnections = @[ +NSArray *const MPPHandMiddleFingerConnections = @[ [[MPPConnection alloc] initWithStart:9 end:10], [[MPPConnection alloc] initWithStart:10 end:11], [[MPPConnection alloc] initWithStart:11 end:12] ]; -NSArray *const kHandRingFingerConnections = @[ +NSArray *const MPPHandRingFingerConnections = @[ [[MPPConnection alloc] initWithStart:13 end:14], [[MPPConnection alloc] initWithStart:14 end:15], [[MPPConnection alloc] initWithStart:15 end:16] ]; -NSArray *const kHandPinkyConnections = @[ +NSArray *const MPPHandPinkyConnections = @[ [[MPPConnection alloc] initWithStart:16 end:17], [[MPPConnection alloc] initWithStart:17 end:18], [[MPPConnection alloc] initWithStart:18 end:19] ]; -NSArray *const kHandConnections = [[[[[[NSArray +NSArray *const MPPHandConnections = [[[[[[NSArray arrayWithArray:kHandPalmConnections] arrayByAddingObjectsFromArray:kHandThumbConnections] arrayByAddingObjectsFromArray:kHandIndexFingerConnections] arrayByAddingObjectsFromArray:kHandMiddleFingerConnections]